Home | History | Annotate | Line # | Download | only in i915
i915_gem.c revision 1.1.1.2
      1 /*
      2  * Copyright  2008 Intel Corporation
      3  *
      4  * Permission is hereby granted, free of charge, to any person obtaining a
      5  * copy of this software and associated documentation files (the "Software"),
      6  * to deal in the Software without restriction, including without limitation
      7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
      8  * and/or sell copies of the Software, and to permit persons to whom the
      9  * Software is furnished to do so, subject to the following conditions:
     10  *
     11  * The above copyright notice and this permission notice (including the next
     12  * paragraph) shall be included in all copies or substantial portions of the
     13  * Software.
     14  *
     15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
     18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
     19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
     20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
     21  * IN THE SOFTWARE.
     22  *
     23  * Authors:
     24  *    Eric Anholt <eric (at) anholt.net>
     25  *
     26  */
     27 
     28 #include <drm/drmP.h>
     29 #include <drm/drm_vma_manager.h>
     30 #include <drm/i915_drm.h>
     31 #include "i915_drv.h"
     32 #include "i915_trace.h"
     33 #include "intel_drv.h"
     34 #include <linux/shmem_fs.h>
     35 #include <linux/slab.h>
     36 #include <linux/swap.h>
     37 #include <linux/pci.h>
     38 #include <linux/dma-buf.h>
     39 
     40 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
     41 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
     42 						   bool force);
     43 static __must_check int
     44 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
     45 			       bool readonly);
     46 
     47 static void i915_gem_write_fence(struct drm_device *dev, int reg,
     48 				 struct drm_i915_gem_object *obj);
     49 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
     50 					 struct drm_i915_fence_reg *fence,
     51 					 bool enable);
     52 
     53 static unsigned long i915_gem_inactive_count(struct shrinker *shrinker,
     54 					     struct shrink_control *sc);
     55 static unsigned long i915_gem_inactive_scan(struct shrinker *shrinker,
     56 					    struct shrink_control *sc);
     57 static unsigned long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
     58 static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
     59 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
     60 static void i915_gem_retire_requests_ring(struct intel_ring_buffer *ring);
     61 
     62 static bool cpu_cache_is_coherent(struct drm_device *dev,
     63 				  enum i915_cache_level level)
     64 {
     65 	return HAS_LLC(dev) || level != I915_CACHE_NONE;
     66 }
     67 
     68 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
     69 {
     70 	if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
     71 		return true;
     72 
     73 	return obj->pin_display;
     74 }
     75 
     76 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
     77 {
     78 	if (obj->tiling_mode)
     79 		i915_gem_release_mmap(obj);
     80 
     81 	/* As we do not have an associated fence register, we will force
     82 	 * a tiling change if we ever need to acquire one.
     83 	 */
     84 	obj->fence_dirty = false;
     85 	obj->fence_reg = I915_FENCE_REG_NONE;
     86 }
     87 
     88 /* some bookkeeping */
     89 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
     90 				  size_t size)
     91 {
     92 	spin_lock(&dev_priv->mm.object_stat_lock);
     93 	dev_priv->mm.object_count++;
     94 	dev_priv->mm.object_memory += size;
     95 	spin_unlock(&dev_priv->mm.object_stat_lock);
     96 }
     97 
     98 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
     99 				     size_t size)
    100 {
    101 	spin_lock(&dev_priv->mm.object_stat_lock);
    102 	dev_priv->mm.object_count--;
    103 	dev_priv->mm.object_memory -= size;
    104 	spin_unlock(&dev_priv->mm.object_stat_lock);
    105 }
    106 
    107 static int
    108 i915_gem_wait_for_error(struct i915_gpu_error *error)
    109 {
    110 	int ret;
    111 
    112 #define EXIT_COND (!i915_reset_in_progress(error) || \
    113 		   i915_terminally_wedged(error))
    114 	if (EXIT_COND)
    115 		return 0;
    116 
    117 	/*
    118 	 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
    119 	 * userspace. If it takes that long something really bad is going on and
    120 	 * we should simply try to bail out and fail as gracefully as possible.
    121 	 */
    122 	ret = wait_event_interruptible_timeout(error->reset_queue,
    123 					       EXIT_COND,
    124 					       10*HZ);
    125 	if (ret == 0) {
    126 		DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
    127 		return -EIO;
    128 	} else if (ret < 0) {
    129 		return ret;
    130 	}
    131 #undef EXIT_COND
    132 
    133 	return 0;
    134 }
    135 
    136 int i915_mutex_lock_interruptible(struct drm_device *dev)
    137 {
    138 	struct drm_i915_private *dev_priv = dev->dev_private;
    139 	int ret;
    140 
    141 	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
    142 	if (ret)
    143 		return ret;
    144 
    145 	ret = mutex_lock_interruptible(&dev->struct_mutex);
    146 	if (ret)
    147 		return ret;
    148 
    149 	WARN_ON(i915_verify_lists(dev));
    150 	return 0;
    151 }
    152 
    153 static inline bool
    154 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
    155 {
    156 	return i915_gem_obj_bound_any(obj) && !obj->active;
    157 }
    158 
    159 int
    160 i915_gem_init_ioctl(struct drm_device *dev, void *data,
    161 		    struct drm_file *file)
    162 {
    163 	struct drm_i915_private *dev_priv = dev->dev_private;
    164 	struct drm_i915_gem_init *args = data;
    165 
    166 	if (drm_core_check_feature(dev, DRIVER_MODESET))
    167 		return -ENODEV;
    168 
    169 	if (args->gtt_start >= args->gtt_end ||
    170 	    (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
    171 		return -EINVAL;
    172 
    173 	/* GEM with user mode setting was never supported on ilk and later. */
    174 	if (INTEL_INFO(dev)->gen >= 5)
    175 		return -ENODEV;
    176 
    177 	mutex_lock(&dev->struct_mutex);
    178 	i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
    179 				  args->gtt_end);
    180 	dev_priv->gtt.mappable_end = args->gtt_end;
    181 	mutex_unlock(&dev->struct_mutex);
    182 
    183 	return 0;
    184 }
    185 
    186 int
    187 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
    188 			    struct drm_file *file)
    189 {
    190 	struct drm_i915_private *dev_priv = dev->dev_private;
    191 	struct drm_i915_gem_get_aperture *args = data;
    192 	struct drm_i915_gem_object *obj;
    193 	size_t pinned;
    194 
    195 	pinned = 0;
    196 	mutex_lock(&dev->struct_mutex);
    197 	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
    198 		if (i915_gem_obj_is_pinned(obj))
    199 			pinned += i915_gem_obj_ggtt_size(obj);
    200 	mutex_unlock(&dev->struct_mutex);
    201 
    202 	args->aper_size = dev_priv->gtt.base.total;
    203 	args->aper_available_size = args->aper_size - pinned;
    204 
    205 	return 0;
    206 }
    207 
    208 static void i915_gem_object_detach_phys(struct drm_i915_gem_object *obj)
    209 {
    210 	drm_dma_handle_t *phys = obj->phys_handle;
    211 
    212 	if (!phys)
    213 		return;
    214 
    215 	if (obj->madv == I915_MADV_WILLNEED) {
    216 		struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
    217 		char *vaddr = phys->vaddr;
    218 		int i;
    219 
    220 		for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
    221 			struct page *page = shmem_read_mapping_page(mapping, i);
    222 			if (!IS_ERR(page)) {
    223 				char *dst = kmap_atomic(page);
    224 				memcpy(dst, vaddr, PAGE_SIZE);
    225 				drm_clflush_virt_range(dst, PAGE_SIZE);
    226 				kunmap_atomic(dst);
    227 
    228 				set_page_dirty(page);
    229 				mark_page_accessed(page);
    230 				page_cache_release(page);
    231 			}
    232 			vaddr += PAGE_SIZE;
    233 		}
    234 		i915_gem_chipset_flush(obj->base.dev);
    235 	}
    236 
    237 #ifdef CONFIG_X86
    238 	set_memory_wb((unsigned long)phys->vaddr, phys->size / PAGE_SIZE);
    239 #endif
    240 	drm_pci_free(obj->base.dev, phys);
    241 	obj->phys_handle = NULL;
    242 }
    243 
    244 int
    245 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
    246 			    int align)
    247 {
    248 	drm_dma_handle_t *phys;
    249 	struct address_space *mapping;
    250 	char *vaddr;
    251 	int i;
    252 
    253 	if (obj->phys_handle) {
    254 		if ((unsigned long)obj->phys_handle->vaddr & (align -1))
    255 			return -EBUSY;
    256 
    257 		return 0;
    258 	}
    259 
    260 	if (obj->madv != I915_MADV_WILLNEED)
    261 		return -EFAULT;
    262 
    263 	if (obj->base.filp == NULL)
    264 		return -EINVAL;
    265 
    266 	/* create a new object */
    267 	phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
    268 	if (!phys)
    269 		return -ENOMEM;
    270 
    271 	vaddr = phys->vaddr;
    272 #ifdef CONFIG_X86
    273 	set_memory_wc((unsigned long)vaddr, phys->size / PAGE_SIZE);
    274 #endif
    275 	mapping = file_inode(obj->base.filp)->i_mapping;
    276 	for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
    277 		struct page *page;
    278 		char *src;
    279 
    280 		page = shmem_read_mapping_page(mapping, i);
    281 		if (IS_ERR(page)) {
    282 #ifdef CONFIG_X86
    283 			set_memory_wb((unsigned long)phys->vaddr, phys->size / PAGE_SIZE);
    284 #endif
    285 			drm_pci_free(obj->base.dev, phys);
    286 			return PTR_ERR(page);
    287 		}
    288 
    289 		src = kmap_atomic(page);
    290 		memcpy(vaddr, src, PAGE_SIZE);
    291 		kunmap_atomic(src);
    292 
    293 		mark_page_accessed(page);
    294 		page_cache_release(page);
    295 
    296 		vaddr += PAGE_SIZE;
    297 	}
    298 
    299 	obj->phys_handle = phys;
    300 	return 0;
    301 }
    302 
    303 static int
    304 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
    305 		     struct drm_i915_gem_pwrite *args,
    306 		     struct drm_file *file_priv)
    307 {
    308 	struct drm_device *dev = obj->base.dev;
    309 	void *vaddr = obj->phys_handle->vaddr + args->offset;
    310 	char __user *user_data = to_user_ptr(args->data_ptr);
    311 
    312 	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
    313 		unsigned long unwritten;
    314 
    315 		/* The physical object once assigned is fixed for the lifetime
    316 		 * of the obj, so we can safely drop the lock and continue
    317 		 * to access vaddr.
    318 		 */
    319 		mutex_unlock(&dev->struct_mutex);
    320 		unwritten = copy_from_user(vaddr, user_data, args->size);
    321 		mutex_lock(&dev->struct_mutex);
    322 		if (unwritten)
    323 			return -EFAULT;
    324 	}
    325 
    326 	i915_gem_chipset_flush(dev);
    327 	return 0;
    328 }
    329 
    330 void *i915_gem_object_alloc(struct drm_device *dev)
    331 {
    332 	struct drm_i915_private *dev_priv = dev->dev_private;
    333 	return kmem_cache_zalloc(dev_priv->slab, GFP_KERNEL);
    334 }
    335 
    336 void i915_gem_object_free(struct drm_i915_gem_object *obj)
    337 {
    338 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
    339 	kmem_cache_free(dev_priv->slab, obj);
    340 }
    341 
    342 static int
    343 i915_gem_create(struct drm_file *file,
    344 		struct drm_device *dev,
    345 		uint64_t size,
    346 		uint32_t *handle_p)
    347 {
    348 	struct drm_i915_gem_object *obj;
    349 	int ret;
    350 	u32 handle;
    351 
    352 	size = roundup(size, PAGE_SIZE);
    353 	if (size == 0)
    354 		return -EINVAL;
    355 
    356 	/* Allocate the new object */
    357 	obj = i915_gem_alloc_object(dev, size);
    358 	if (obj == NULL)
    359 		return -ENOMEM;
    360 
    361 	ret = drm_gem_handle_create(file, &obj->base, &handle);
    362 	/* drop reference from allocate - handle holds it now */
    363 	drm_gem_object_unreference_unlocked(&obj->base);
    364 	if (ret)
    365 		return ret;
    366 
    367 	*handle_p = handle;
    368 	return 0;
    369 }
    370 
    371 int
    372 i915_gem_dumb_create(struct drm_file *file,
    373 		     struct drm_device *dev,
    374 		     struct drm_mode_create_dumb *args)
    375 {
    376 	/* have to work out size/pitch and return them */
    377 	args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
    378 	args->size = args->pitch * args->height;
    379 	return i915_gem_create(file, dev,
    380 			       args->size, &args->handle);
    381 }
    382 
    383 /**
    384  * Creates a new mm object and returns a handle to it.
    385  */
    386 int
    387 i915_gem_create_ioctl(struct drm_device *dev, void *data,
    388 		      struct drm_file *file)
    389 {
    390 	struct drm_i915_gem_create *args = data;
    391 
    392 	return i915_gem_create(file, dev,
    393 			       args->size, &args->handle);
    394 }
    395 
    396 static inline int
    397 __copy_to_user_swizzled(char __user *cpu_vaddr,
    398 			const char *gpu_vaddr, int gpu_offset,
    399 			int length)
    400 {
    401 	int ret, cpu_offset = 0;
    402 
    403 	while (length > 0) {
    404 		int cacheline_end = ALIGN(gpu_offset + 1, 64);
    405 		int this_length = min(cacheline_end - gpu_offset, length);
    406 		int swizzled_gpu_offset = gpu_offset ^ 64;
    407 
    408 		ret = __copy_to_user(cpu_vaddr + cpu_offset,
    409 				     gpu_vaddr + swizzled_gpu_offset,
    410 				     this_length);
    411 		if (ret)
    412 			return ret + length;
    413 
    414 		cpu_offset += this_length;
    415 		gpu_offset += this_length;
    416 		length -= this_length;
    417 	}
    418 
    419 	return 0;
    420 }
    421 
    422 static inline int
    423 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
    424 			  const char __user *cpu_vaddr,
    425 			  int length)
    426 {
    427 	int ret, cpu_offset = 0;
    428 
    429 	while (length > 0) {
    430 		int cacheline_end = ALIGN(gpu_offset + 1, 64);
    431 		int this_length = min(cacheline_end - gpu_offset, length);
    432 		int swizzled_gpu_offset = gpu_offset ^ 64;
    433 
    434 		ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
    435 				       cpu_vaddr + cpu_offset,
    436 				       this_length);
    437 		if (ret)
    438 			return ret + length;
    439 
    440 		cpu_offset += this_length;
    441 		gpu_offset += this_length;
    442 		length -= this_length;
    443 	}
    444 
    445 	return 0;
    446 }
    447 
    448 /*
    449  * Pins the specified object's pages and synchronizes the object with
    450  * GPU accesses. Sets needs_clflush to non-zero if the caller should
    451  * flush the object from the CPU cache.
    452  */
    453 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
    454 				    int *needs_clflush)
    455 {
    456 	int ret;
    457 
    458 	*needs_clflush = 0;
    459 
    460 	if (!obj->base.filp)
    461 		return -EINVAL;
    462 
    463 	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
    464 		/* If we're not in the cpu read domain, set ourself into the gtt
    465 		 * read domain and manually flush cachelines (if required). This
    466 		 * optimizes for the case when the gpu will dirty the data
    467 		 * anyway again before the next pread happens. */
    468 		*needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
    469 							obj->cache_level);
    470 		ret = i915_gem_object_wait_rendering(obj, true);
    471 		if (ret)
    472 			return ret;
    473 	}
    474 
    475 	ret = i915_gem_object_get_pages(obj);
    476 	if (ret)
    477 		return ret;
    478 
    479 	i915_gem_object_pin_pages(obj);
    480 
    481 	return ret;
    482 }
    483 
    484 /* Per-page copy function for the shmem pread fastpath.
    485  * Flushes invalid cachelines before reading the target if
    486  * needs_clflush is set. */
    487 static int
    488 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
    489 		 char __user *user_data,
    490 		 bool page_do_bit17_swizzling, bool needs_clflush)
    491 {
    492 	char *vaddr;
    493 	int ret;
    494 
    495 	if (unlikely(page_do_bit17_swizzling))
    496 		return -EINVAL;
    497 
    498 	vaddr = kmap_atomic(page);
    499 	if (needs_clflush)
    500 		drm_clflush_virt_range(vaddr + shmem_page_offset,
    501 				       page_length);
    502 	ret = __copy_to_user_inatomic(user_data,
    503 				      vaddr + shmem_page_offset,
    504 				      page_length);
    505 	kunmap_atomic(vaddr);
    506 
    507 	return ret ? -EFAULT : 0;
    508 }
    509 
    510 static void
    511 shmem_clflush_swizzled_range(char *addr, unsigned long length,
    512 			     bool swizzled)
    513 {
    514 	if (unlikely(swizzled)) {
    515 		unsigned long start = (unsigned long) addr;
    516 		unsigned long end = (unsigned long) addr + length;
    517 
    518 		/* For swizzling simply ensure that we always flush both
    519 		 * channels. Lame, but simple and it works. Swizzled
    520 		 * pwrite/pread is far from a hotpath - current userspace
    521 		 * doesn't use it at all. */
    522 		start = round_down(start, 128);
    523 		end = round_up(end, 128);
    524 
    525 		drm_clflush_virt_range((void *)start, end - start);
    526 	} else {
    527 		drm_clflush_virt_range(addr, length);
    528 	}
    529 
    530 }
    531 
    532 /* Only difference to the fast-path function is that this can handle bit17
    533  * and uses non-atomic copy and kmap functions. */
    534 static int
    535 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
    536 		 char __user *user_data,
    537 		 bool page_do_bit17_swizzling, bool needs_clflush)
    538 {
    539 	char *vaddr;
    540 	int ret;
    541 
    542 	vaddr = kmap(page);
    543 	if (needs_clflush)
    544 		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
    545 					     page_length,
    546 					     page_do_bit17_swizzling);
    547 
    548 	if (page_do_bit17_swizzling)
    549 		ret = __copy_to_user_swizzled(user_data,
    550 					      vaddr, shmem_page_offset,
    551 					      page_length);
    552 	else
    553 		ret = __copy_to_user(user_data,
    554 				     vaddr + shmem_page_offset,
    555 				     page_length);
    556 	kunmap(page);
    557 
    558 	return ret ? - EFAULT : 0;
    559 }
    560 
    561 static int
    562 i915_gem_shmem_pread(struct drm_device *dev,
    563 		     struct drm_i915_gem_object *obj,
    564 		     struct drm_i915_gem_pread *args,
    565 		     struct drm_file *file)
    566 {
    567 	char __user *user_data;
    568 	ssize_t remain;
    569 	loff_t offset;
    570 	int shmem_page_offset, page_length, ret = 0;
    571 	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
    572 	int prefaulted = 0;
    573 	int needs_clflush = 0;
    574 	struct sg_page_iter sg_iter;
    575 
    576 	user_data = to_user_ptr(args->data_ptr);
    577 	remain = args->size;
    578 
    579 	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
    580 
    581 	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
    582 	if (ret)
    583 		return ret;
    584 
    585 	offset = args->offset;
    586 
    587 	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
    588 			 offset >> PAGE_SHIFT) {
    589 		struct page *page = sg_page_iter_page(&sg_iter);
    590 
    591 		if (remain <= 0)
    592 			break;
    593 
    594 		/* Operation in this page
    595 		 *
    596 		 * shmem_page_offset = offset within page in shmem file
    597 		 * page_length = bytes to copy for this page
    598 		 */
    599 		shmem_page_offset = offset_in_page(offset);
    600 		page_length = remain;
    601 		if ((shmem_page_offset + page_length) > PAGE_SIZE)
    602 			page_length = PAGE_SIZE - shmem_page_offset;
    603 
    604 		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
    605 			(page_to_phys(page) & (1 << 17)) != 0;
    606 
    607 		ret = shmem_pread_fast(page, shmem_page_offset, page_length,
    608 				       user_data, page_do_bit17_swizzling,
    609 				       needs_clflush);
    610 		if (ret == 0)
    611 			goto next_page;
    612 
    613 		mutex_unlock(&dev->struct_mutex);
    614 
    615 		if (likely(!i915.prefault_disable) && !prefaulted) {
    616 			ret = fault_in_multipages_writeable(user_data, remain);
    617 			/* Userspace is tricking us, but we've already clobbered
    618 			 * its pages with the prefault and promised to write the
    619 			 * data up to the first fault. Hence ignore any errors
    620 			 * and just continue. */
    621 			(void)ret;
    622 			prefaulted = 1;
    623 		}
    624 
    625 		ret = shmem_pread_slow(page, shmem_page_offset, page_length,
    626 				       user_data, page_do_bit17_swizzling,
    627 				       needs_clflush);
    628 
    629 		mutex_lock(&dev->struct_mutex);
    630 
    631 		if (ret)
    632 			goto out;
    633 
    634 next_page:
    635 		remain -= page_length;
    636 		user_data += page_length;
    637 		offset += page_length;
    638 	}
    639 
    640 out:
    641 	i915_gem_object_unpin_pages(obj);
    642 
    643 	return ret;
    644 }
    645 
    646 /**
    647  * Reads data from the object referenced by handle.
    648  *
    649  * On error, the contents of *data are undefined.
    650  */
    651 int
    652 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
    653 		     struct drm_file *file)
    654 {
    655 	struct drm_i915_gem_pread *args = data;
    656 	struct drm_i915_gem_object *obj;
    657 	int ret = 0;
    658 
    659 	if (args->size == 0)
    660 		return 0;
    661 
    662 	if (!access_ok(VERIFY_WRITE,
    663 		       to_user_ptr(args->data_ptr),
    664 		       args->size))
    665 		return -EFAULT;
    666 
    667 	ret = i915_mutex_lock_interruptible(dev);
    668 	if (ret)
    669 		return ret;
    670 
    671 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
    672 	if (&obj->base == NULL) {
    673 		ret = -ENOENT;
    674 		goto unlock;
    675 	}
    676 
    677 	/* Bounds check source.  */
    678 	if (args->offset > obj->base.size ||
    679 	    args->size > obj->base.size - args->offset) {
    680 		ret = -EINVAL;
    681 		goto out;
    682 	}
    683 
    684 	/* prime objects have no backing filp to GEM pread/pwrite
    685 	 * pages from.
    686 	 */
    687 	if (!obj->base.filp) {
    688 		ret = -EINVAL;
    689 		goto out;
    690 	}
    691 
    692 	trace_i915_gem_object_pread(obj, args->offset, args->size);
    693 
    694 	ret = i915_gem_shmem_pread(dev, obj, args, file);
    695 
    696 out:
    697 	drm_gem_object_unreference(&obj->base);
    698 unlock:
    699 	mutex_unlock(&dev->struct_mutex);
    700 	return ret;
    701 }
    702 
    703 /* This is the fast write path which cannot handle
    704  * page faults in the source data
    705  */
    706 
    707 static inline int
    708 fast_user_write(struct io_mapping *mapping,
    709 		loff_t page_base, int page_offset,
    710 		char __user *user_data,
    711 		int length)
    712 {
    713 	void __iomem *vaddr_atomic;
    714 	void *vaddr;
    715 	unsigned long unwritten;
    716 
    717 	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
    718 	/* We can use the cpu mem copy function because this is X86. */
    719 	vaddr = (void __force*)vaddr_atomic + page_offset;
    720 	unwritten = __copy_from_user_inatomic_nocache(vaddr,
    721 						      user_data, length);
    722 	io_mapping_unmap_atomic(vaddr_atomic);
    723 	return unwritten;
    724 }
    725 
    726 /**
    727  * This is the fast pwrite path, where we copy the data directly from the
    728  * user into the GTT, uncached.
    729  */
    730 static int
    731 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
    732 			 struct drm_i915_gem_object *obj,
    733 			 struct drm_i915_gem_pwrite *args,
    734 			 struct drm_file *file)
    735 {
    736 	struct drm_i915_private *dev_priv = dev->dev_private;
    737 	ssize_t remain;
    738 	loff_t offset, page_base;
    739 	char __user *user_data;
    740 	int page_offset, page_length, ret;
    741 
    742 	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
    743 	if (ret)
    744 		goto out;
    745 
    746 	ret = i915_gem_object_set_to_gtt_domain(obj, true);
    747 	if (ret)
    748 		goto out_unpin;
    749 
    750 	ret = i915_gem_object_put_fence(obj);
    751 	if (ret)
    752 		goto out_unpin;
    753 
    754 	user_data = to_user_ptr(args->data_ptr);
    755 	remain = args->size;
    756 
    757 	offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
    758 
    759 	while (remain > 0) {
    760 		/* Operation in this page
    761 		 *
    762 		 * page_base = page offset within aperture
    763 		 * page_offset = offset within page
    764 		 * page_length = bytes to copy for this page
    765 		 */
    766 		page_base = offset & PAGE_MASK;
    767 		page_offset = offset_in_page(offset);
    768 		page_length = remain;
    769 		if ((page_offset + remain) > PAGE_SIZE)
    770 			page_length = PAGE_SIZE - page_offset;
    771 
    772 		/* If we get a fault while copying data, then (presumably) our
    773 		 * source page isn't available.  Return the error and we'll
    774 		 * retry in the slow path.
    775 		 */
    776 		if (fast_user_write(dev_priv->gtt.mappable, page_base,
    777 				    page_offset, user_data, page_length)) {
    778 			ret = -EFAULT;
    779 			goto out_unpin;
    780 		}
    781 
    782 		remain -= page_length;
    783 		user_data += page_length;
    784 		offset += page_length;
    785 	}
    786 
    787 out_unpin:
    788 	i915_gem_object_ggtt_unpin(obj);
    789 out:
    790 	return ret;
    791 }
    792 
    793 /* Per-page copy function for the shmem pwrite fastpath.
    794  * Flushes invalid cachelines before writing to the target if
    795  * needs_clflush_before is set and flushes out any written cachelines after
    796  * writing if needs_clflush is set. */
    797 static int
    798 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
    799 		  char __user *user_data,
    800 		  bool page_do_bit17_swizzling,
    801 		  bool needs_clflush_before,
    802 		  bool needs_clflush_after)
    803 {
    804 	char *vaddr;
    805 	int ret;
    806 
    807 	if (unlikely(page_do_bit17_swizzling))
    808 		return -EINVAL;
    809 
    810 	vaddr = kmap_atomic(page);
    811 	if (needs_clflush_before)
    812 		drm_clflush_virt_range(vaddr + shmem_page_offset,
    813 				       page_length);
    814 	ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
    815 					user_data, page_length);
    816 	if (needs_clflush_after)
    817 		drm_clflush_virt_range(vaddr + shmem_page_offset,
    818 				       page_length);
    819 	kunmap_atomic(vaddr);
    820 
    821 	return ret ? -EFAULT : 0;
    822 }
    823 
    824 /* Only difference to the fast-path function is that this can handle bit17
    825  * and uses non-atomic copy and kmap functions. */
    826 static int
    827 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
    828 		  char __user *user_data,
    829 		  bool page_do_bit17_swizzling,
    830 		  bool needs_clflush_before,
    831 		  bool needs_clflush_after)
    832 {
    833 	char *vaddr;
    834 	int ret;
    835 
    836 	vaddr = kmap(page);
    837 	if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
    838 		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
    839 					     page_length,
    840 					     page_do_bit17_swizzling);
    841 	if (page_do_bit17_swizzling)
    842 		ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
    843 						user_data,
    844 						page_length);
    845 	else
    846 		ret = __copy_from_user(vaddr + shmem_page_offset,
    847 				       user_data,
    848 				       page_length);
    849 	if (needs_clflush_after)
    850 		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
    851 					     page_length,
    852 					     page_do_bit17_swizzling);
    853 	kunmap(page);
    854 
    855 	return ret ? -EFAULT : 0;
    856 }
    857 
    858 static int
    859 i915_gem_shmem_pwrite(struct drm_device *dev,
    860 		      struct drm_i915_gem_object *obj,
    861 		      struct drm_i915_gem_pwrite *args,
    862 		      struct drm_file *file)
    863 {
    864 	ssize_t remain;
    865 	loff_t offset;
    866 	char __user *user_data;
    867 	int shmem_page_offset, page_length, ret = 0;
    868 	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
    869 	int hit_slowpath = 0;
    870 	int needs_clflush_after = 0;
    871 	int needs_clflush_before = 0;
    872 	struct sg_page_iter sg_iter;
    873 
    874 	user_data = to_user_ptr(args->data_ptr);
    875 	remain = args->size;
    876 
    877 	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
    878 
    879 	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
    880 		/* If we're not in the cpu write domain, set ourself into the gtt
    881 		 * write domain and manually flush cachelines (if required). This
    882 		 * optimizes for the case when the gpu will use the data
    883 		 * right away and we therefore have to clflush anyway. */
    884 		needs_clflush_after = cpu_write_needs_clflush(obj);
    885 		ret = i915_gem_object_wait_rendering(obj, false);
    886 		if (ret)
    887 			return ret;
    888 	}
    889 	/* Same trick applies to invalidate partially written cachelines read
    890 	 * before writing. */
    891 	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
    892 		needs_clflush_before =
    893 			!cpu_cache_is_coherent(dev, obj->cache_level);
    894 
    895 	ret = i915_gem_object_get_pages(obj);
    896 	if (ret)
    897 		return ret;
    898 
    899 	i915_gem_object_pin_pages(obj);
    900 
    901 	offset = args->offset;
    902 	obj->dirty = 1;
    903 
    904 	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
    905 			 offset >> PAGE_SHIFT) {
    906 		struct page *page = sg_page_iter_page(&sg_iter);
    907 		int partial_cacheline_write;
    908 
    909 		if (remain <= 0)
    910 			break;
    911 
    912 		/* Operation in this page
    913 		 *
    914 		 * shmem_page_offset = offset within page in shmem file
    915 		 * page_length = bytes to copy for this page
    916 		 */
    917 		shmem_page_offset = offset_in_page(offset);
    918 
    919 		page_length = remain;
    920 		if ((shmem_page_offset + page_length) > PAGE_SIZE)
    921 			page_length = PAGE_SIZE - shmem_page_offset;
    922 
    923 		/* If we don't overwrite a cacheline completely we need to be
    924 		 * careful to have up-to-date data by first clflushing. Don't
    925 		 * overcomplicate things and flush the entire patch. */
    926 		partial_cacheline_write = needs_clflush_before &&
    927 			((shmem_page_offset | page_length)
    928 				& (boot_cpu_data.x86_clflush_size - 1));
    929 
    930 		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
    931 			(page_to_phys(page) & (1 << 17)) != 0;
    932 
    933 		ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
    934 					user_data, page_do_bit17_swizzling,
    935 					partial_cacheline_write,
    936 					needs_clflush_after);
    937 		if (ret == 0)
    938 			goto next_page;
    939 
    940 		hit_slowpath = 1;
    941 		mutex_unlock(&dev->struct_mutex);
    942 		ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
    943 					user_data, page_do_bit17_swizzling,
    944 					partial_cacheline_write,
    945 					needs_clflush_after);
    946 
    947 		mutex_lock(&dev->struct_mutex);
    948 
    949 		if (ret)
    950 			goto out;
    951 
    952 next_page:
    953 		remain -= page_length;
    954 		user_data += page_length;
    955 		offset += page_length;
    956 	}
    957 
    958 out:
    959 	i915_gem_object_unpin_pages(obj);
    960 
    961 	if (hit_slowpath) {
    962 		/*
    963 		 * Fixup: Flush cpu caches in case we didn't flush the dirty
    964 		 * cachelines in-line while writing and the object moved
    965 		 * out of the cpu write domain while we've dropped the lock.
    966 		 */
    967 		if (!needs_clflush_after &&
    968 		    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
    969 			if (i915_gem_clflush_object(obj, obj->pin_display))
    970 				i915_gem_chipset_flush(dev);
    971 		}
    972 	}
    973 
    974 	if (needs_clflush_after)
    975 		i915_gem_chipset_flush(dev);
    976 
    977 	return ret;
    978 }
    979 
    980 /**
    981  * Writes data to the object referenced by handle.
    982  *
    983  * On error, the contents of the buffer that were to be modified are undefined.
    984  */
    985 int
    986 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
    987 		      struct drm_file *file)
    988 {
    989 	struct drm_i915_gem_pwrite *args = data;
    990 	struct drm_i915_gem_object *obj;
    991 	int ret;
    992 
    993 	if (args->size == 0)
    994 		return 0;
    995 
    996 	if (!access_ok(VERIFY_READ,
    997 		       to_user_ptr(args->data_ptr),
    998 		       args->size))
    999 		return -EFAULT;
   1000 
   1001 	if (likely(!i915.prefault_disable)) {
   1002 		ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
   1003 						   args->size);
   1004 		if (ret)
   1005 			return -EFAULT;
   1006 	}
   1007 
   1008 	ret = i915_mutex_lock_interruptible(dev);
   1009 	if (ret)
   1010 		return ret;
   1011 
   1012 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
   1013 	if (&obj->base == NULL) {
   1014 		ret = -ENOENT;
   1015 		goto unlock;
   1016 	}
   1017 
   1018 	/* Bounds check destination. */
   1019 	if (args->offset > obj->base.size ||
   1020 	    args->size > obj->base.size - args->offset) {
   1021 		ret = -EINVAL;
   1022 		goto out;
   1023 	}
   1024 
   1025 	/* prime objects have no backing filp to GEM pread/pwrite
   1026 	 * pages from.
   1027 	 */
   1028 	if (!obj->base.filp) {
   1029 		ret = -EINVAL;
   1030 		goto out;
   1031 	}
   1032 
   1033 	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
   1034 
   1035 	ret = -EFAULT;
   1036 	/* We can only do the GTT pwrite on untiled buffers, as otherwise
   1037 	 * it would end up going through the fenced access, and we'll get
   1038 	 * different detiling behavior between reading and writing.
   1039 	 * pread/pwrite currently are reading and writing from the CPU
   1040 	 * perspective, requiring manual detiling by the client.
   1041 	 */
   1042 	if (obj->phys_handle) {
   1043 		ret = i915_gem_phys_pwrite(obj, args, file);
   1044 		goto out;
   1045 	}
   1046 
   1047 	if (obj->tiling_mode == I915_TILING_NONE &&
   1048 	    obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
   1049 	    cpu_write_needs_clflush(obj)) {
   1050 		ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
   1051 		/* Note that the gtt paths might fail with non-page-backed user
   1052 		 * pointers (e.g. gtt mappings when moving data between
   1053 		 * textures). Fallback to the shmem path in that case. */
   1054 	}
   1055 
   1056 	if (ret == -EFAULT || ret == -ENOSPC)
   1057 		ret = i915_gem_shmem_pwrite(dev, obj, args, file);
   1058 
   1059 out:
   1060 	drm_gem_object_unreference(&obj->base);
   1061 unlock:
   1062 	mutex_unlock(&dev->struct_mutex);
   1063 	return ret;
   1064 }
   1065 
   1066 int
   1067 i915_gem_check_wedge(struct i915_gpu_error *error,
   1068 		     bool interruptible)
   1069 {
   1070 	if (i915_reset_in_progress(error)) {
   1071 		/* Non-interruptible callers can't handle -EAGAIN, hence return
   1072 		 * -EIO unconditionally for these. */
   1073 		if (!interruptible)
   1074 			return -EIO;
   1075 
   1076 		/* Recovery complete, but the reset failed ... */
   1077 		if (i915_terminally_wedged(error))
   1078 			return -EIO;
   1079 
   1080 		return -EAGAIN;
   1081 	}
   1082 
   1083 	return 0;
   1084 }
   1085 
   1086 /*
   1087  * Compare seqno against outstanding lazy request. Emit a request if they are
   1088  * equal.
   1089  */
   1090 static int
   1091 i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
   1092 {
   1093 	int ret;
   1094 
   1095 	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
   1096 
   1097 	ret = 0;
   1098 	if (seqno == ring->outstanding_lazy_seqno)
   1099 		ret = i915_add_request(ring, NULL);
   1100 
   1101 	return ret;
   1102 }
   1103 
   1104 static void fake_irq(unsigned long data)
   1105 {
   1106 	wake_up_process((struct task_struct *)data);
   1107 }
   1108 
   1109 static bool missed_irq(struct drm_i915_private *dev_priv,
   1110 		       struct intel_ring_buffer *ring)
   1111 {
   1112 	return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
   1113 }
   1114 
   1115 static bool can_wait_boost(struct drm_i915_file_private *file_priv)
   1116 {
   1117 	if (file_priv == NULL)
   1118 		return true;
   1119 
   1120 	return !atomic_xchg(&file_priv->rps_wait_boost, true);
   1121 }
   1122 
   1123 /**
   1124  * __wait_seqno - wait until execution of seqno has finished
   1125  * @ring: the ring expected to report seqno
   1126  * @seqno: duh!
   1127  * @reset_counter: reset sequence associated with the given seqno
   1128  * @interruptible: do an interruptible wait (normally yes)
   1129  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
   1130  *
   1131  * Note: It is of utmost importance that the passed in seqno and reset_counter
   1132  * values have been read by the caller in an smp safe manner. Where read-side
   1133  * locks are involved, it is sufficient to read the reset_counter before
   1134  * unlocking the lock that protects the seqno. For lockless tricks, the
   1135  * reset_counter _must_ be read before, and an appropriate smp_rmb must be
   1136  * inserted.
   1137  *
   1138  * Returns 0 if the seqno was found within the alloted time. Else returns the
   1139  * errno with remaining time filled in timeout argument.
   1140  */
   1141 static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
   1142 			unsigned reset_counter,
   1143 			bool interruptible,
   1144 			struct timespec *timeout,
   1145 			struct drm_i915_file_private *file_priv)
   1146 {
   1147 	struct drm_device *dev = ring->dev;
   1148 	struct drm_i915_private *dev_priv = dev->dev_private;
   1149 	const bool irq_test_in_progress =
   1150 		ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
   1151 	struct timespec before, now;
   1152 	DEFINE_WAIT(wait);
   1153 	unsigned long timeout_expire;
   1154 	int ret;
   1155 
   1156 	WARN(dev_priv->pm.irqs_disabled, "IRQs disabled\n");
   1157 
   1158 	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
   1159 		return 0;
   1160 
   1161 	timeout_expire = timeout ? jiffies + timespec_to_jiffies_timeout(timeout) : 0;
   1162 
   1163 	if (INTEL_INFO(dev)->gen >= 6 && can_wait_boost(file_priv)) {
   1164 		gen6_rps_boost(dev_priv);
   1165 		if (file_priv)
   1166 			mod_delayed_work(dev_priv->wq,
   1167 					 &file_priv->mm.idle_work,
   1168 					 msecs_to_jiffies(100));
   1169 	}
   1170 
   1171 	if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring)))
   1172 		return -ENODEV;
   1173 
   1174 	/* Record current time in case interrupted by signal, or wedged */
   1175 	trace_i915_gem_request_wait_begin(ring, seqno);
   1176 	getrawmonotonic(&before);
   1177 	for (;;) {
   1178 		struct timer_list timer;
   1179 
   1180 		prepare_to_wait(&ring->irq_queue, &wait,
   1181 				interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE);
   1182 
   1183 		/* We need to check whether any gpu reset happened in between
   1184 		 * the caller grabbing the seqno and now ... */
   1185 		if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
   1186 			/* ... but upgrade the -EAGAIN to an -EIO if the gpu
   1187 			 * is truely gone. */
   1188 			ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
   1189 			if (ret == 0)
   1190 				ret = -EAGAIN;
   1191 			break;
   1192 		}
   1193 
   1194 		if (i915_seqno_passed(ring->get_seqno(ring, false), seqno)) {
   1195 			ret = 0;
   1196 			break;
   1197 		}
   1198 
   1199 		if (interruptible && signal_pending(current)) {
   1200 			ret = -ERESTARTSYS;
   1201 			break;
   1202 		}
   1203 
   1204 		if (timeout && time_after_eq(jiffies, timeout_expire)) {
   1205 			ret = -ETIME;
   1206 			break;
   1207 		}
   1208 
   1209 		timer.function = NULL;
   1210 		if (timeout || missed_irq(dev_priv, ring)) {
   1211 			unsigned long expire;
   1212 
   1213 			setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
   1214 			expire = missed_irq(dev_priv, ring) ? jiffies + 1 : timeout_expire;
   1215 			mod_timer(&timer, expire);
   1216 		}
   1217 
   1218 		io_schedule();
   1219 
   1220 		if (timer.function) {
   1221 			del_singleshot_timer_sync(&timer);
   1222 			destroy_timer_on_stack(&timer);
   1223 		}
   1224 	}
   1225 	getrawmonotonic(&now);
   1226 	trace_i915_gem_request_wait_end(ring, seqno);
   1227 
   1228 	if (!irq_test_in_progress)
   1229 		ring->irq_put(ring);
   1230 
   1231 	finish_wait(&ring->irq_queue, &wait);
   1232 
   1233 	if (timeout) {
   1234 		struct timespec sleep_time = timespec_sub(now, before);
   1235 		*timeout = timespec_sub(*timeout, sleep_time);
   1236 		if (!timespec_valid(timeout)) /* i.e. negative time remains */
   1237 			set_normalized_timespec(timeout, 0, 0);
   1238 	}
   1239 
   1240 	return ret;
   1241 }
   1242 
   1243 /**
   1244  * Waits for a sequence number to be signaled, and cleans up the
   1245  * request and object lists appropriately for that event.
   1246  */
   1247 int
   1248 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
   1249 {
   1250 	struct drm_device *dev = ring->dev;
   1251 	struct drm_i915_private *dev_priv = dev->dev_private;
   1252 	bool interruptible = dev_priv->mm.interruptible;
   1253 	int ret;
   1254 
   1255 	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
   1256 	BUG_ON(seqno == 0);
   1257 
   1258 	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
   1259 	if (ret)
   1260 		return ret;
   1261 
   1262 	ret = i915_gem_check_olr(ring, seqno);
   1263 	if (ret)
   1264 		return ret;
   1265 
   1266 	return __wait_seqno(ring, seqno,
   1267 			    atomic_read(&dev_priv->gpu_error.reset_counter),
   1268 			    interruptible, NULL, NULL);
   1269 }
   1270 
   1271 static int
   1272 i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
   1273 				     struct intel_ring_buffer *ring)
   1274 {
   1275 	i915_gem_retire_requests_ring(ring);
   1276 
   1277 	/* Manually manage the write flush as we may have not yet
   1278 	 * retired the buffer.
   1279 	 *
   1280 	 * Note that the last_write_seqno is always the earlier of
   1281 	 * the two (read/write) seqno, so if we haved successfully waited,
   1282 	 * we know we have passed the last write.
   1283 	 */
   1284 	obj->last_write_seqno = 0;
   1285 	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
   1286 
   1287 	return 0;
   1288 }
   1289 
   1290 /**
   1291  * Ensures that all rendering to the object has completed and the object is
   1292  * safe to unbind from the GTT or access from the CPU.
   1293  */
   1294 static __must_check int
   1295 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
   1296 			       bool readonly)
   1297 {
   1298 	struct intel_ring_buffer *ring = obj->ring;
   1299 	u32 seqno;
   1300 	int ret;
   1301 
   1302 	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
   1303 	if (seqno == 0)
   1304 		return 0;
   1305 
   1306 	ret = i915_wait_seqno(ring, seqno);
   1307 	if (ret)
   1308 		return ret;
   1309 
   1310 	return i915_gem_object_wait_rendering__tail(obj, ring);
   1311 }
   1312 
   1313 /* A nonblocking variant of the above wait. This is a highly dangerous routine
   1314  * as the object state may change during this call.
   1315  */
   1316 static __must_check int
   1317 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
   1318 					    struct drm_i915_file_private *file_priv,
   1319 					    bool readonly)
   1320 {
   1321 	struct drm_device *dev = obj->base.dev;
   1322 	struct drm_i915_private *dev_priv = dev->dev_private;
   1323 	struct intel_ring_buffer *ring = obj->ring;
   1324 	unsigned reset_counter;
   1325 	u32 seqno;
   1326 	int ret;
   1327 
   1328 	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
   1329 	BUG_ON(!dev_priv->mm.interruptible);
   1330 
   1331 	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
   1332 	if (seqno == 0)
   1333 		return 0;
   1334 
   1335 	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
   1336 	if (ret)
   1337 		return ret;
   1338 
   1339 	ret = i915_gem_check_olr(ring, seqno);
   1340 	if (ret)
   1341 		return ret;
   1342 
   1343 	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
   1344 	mutex_unlock(&dev->struct_mutex);
   1345 	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, file_priv);
   1346 	mutex_lock(&dev->struct_mutex);
   1347 	if (ret)
   1348 		return ret;
   1349 
   1350 	return i915_gem_object_wait_rendering__tail(obj, ring);
   1351 }
   1352 
   1353 /**
   1354  * Called when user space prepares to use an object with the CPU, either
   1355  * through the mmap ioctl's mapping or a GTT mapping.
   1356  */
   1357 int
   1358 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
   1359 			  struct drm_file *file)
   1360 {
   1361 	struct drm_i915_gem_set_domain *args = data;
   1362 	struct drm_i915_gem_object *obj;
   1363 	uint32_t read_domains = args->read_domains;
   1364 	uint32_t write_domain = args->write_domain;
   1365 	int ret;
   1366 
   1367 	/* Only handle setting domains to types used by the CPU. */
   1368 	if (write_domain & I915_GEM_GPU_DOMAINS)
   1369 		return -EINVAL;
   1370 
   1371 	if (read_domains & I915_GEM_GPU_DOMAINS)
   1372 		return -EINVAL;
   1373 
   1374 	/* Having something in the write domain implies it's in the read
   1375 	 * domain, and only that read domain.  Enforce that in the request.
   1376 	 */
   1377 	if (write_domain != 0 && read_domains != write_domain)
   1378 		return -EINVAL;
   1379 
   1380 	ret = i915_mutex_lock_interruptible(dev);
   1381 	if (ret)
   1382 		return ret;
   1383 
   1384 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
   1385 	if (&obj->base == NULL) {
   1386 		ret = -ENOENT;
   1387 		goto unlock;
   1388 	}
   1389 
   1390 	/* Try to flush the object off the GPU without holding the lock.
   1391 	 * We will repeat the flush holding the lock in the normal manner
   1392 	 * to catch cases where we are gazumped.
   1393 	 */
   1394 	ret = i915_gem_object_wait_rendering__nonblocking(obj,
   1395 							  file->driver_priv,
   1396 							  !write_domain);
   1397 	if (ret)
   1398 		goto unref;
   1399 
   1400 	if (read_domains & I915_GEM_DOMAIN_GTT) {
   1401 		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
   1402 
   1403 		/* Silently promote "you're not bound, there was nothing to do"
   1404 		 * to success, since the client was just asking us to
   1405 		 * make sure everything was done.
   1406 		 */
   1407 		if (ret == -EINVAL)
   1408 			ret = 0;
   1409 	} else {
   1410 		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
   1411 	}
   1412 
   1413 unref:
   1414 	drm_gem_object_unreference(&obj->base);
   1415 unlock:
   1416 	mutex_unlock(&dev->struct_mutex);
   1417 	return ret;
   1418 }
   1419 
   1420 /**
   1421  * Called when user space has done writes to this buffer
   1422  */
   1423 int
   1424 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
   1425 			 struct drm_file *file)
   1426 {
   1427 	struct drm_i915_gem_sw_finish *args = data;
   1428 	struct drm_i915_gem_object *obj;
   1429 	int ret = 0;
   1430 
   1431 	ret = i915_mutex_lock_interruptible(dev);
   1432 	if (ret)
   1433 		return ret;
   1434 
   1435 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
   1436 	if (&obj->base == NULL) {
   1437 		ret = -ENOENT;
   1438 		goto unlock;
   1439 	}
   1440 
   1441 	/* Pinned buffers may be scanout, so flush the cache */
   1442 	if (obj->pin_display)
   1443 		i915_gem_object_flush_cpu_write_domain(obj, true);
   1444 
   1445 	drm_gem_object_unreference(&obj->base);
   1446 unlock:
   1447 	mutex_unlock(&dev->struct_mutex);
   1448 	return ret;
   1449 }
   1450 
   1451 /**
   1452  * Maps the contents of an object, returning the address it is mapped
   1453  * into.
   1454  *
   1455  * While the mapping holds a reference on the contents of the object, it doesn't
   1456  * imply a ref on the object itself.
   1457  */
   1458 int
   1459 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
   1460 		    struct drm_file *file)
   1461 {
   1462 	struct drm_i915_gem_mmap *args = data;
   1463 	struct drm_gem_object *obj;
   1464 	unsigned long addr;
   1465 
   1466 	obj = drm_gem_object_lookup(dev, file, args->handle);
   1467 	if (obj == NULL)
   1468 		return -ENOENT;
   1469 
   1470 	/* prime objects have no backing filp to GEM mmap
   1471 	 * pages from.
   1472 	 */
   1473 	if (!obj->filp) {
   1474 		drm_gem_object_unreference_unlocked(obj);
   1475 		return -EINVAL;
   1476 	}
   1477 
   1478 	addr = vm_mmap(obj->filp, 0, args->size,
   1479 		       PROT_READ | PROT_WRITE, MAP_SHARED,
   1480 		       args->offset);
   1481 	drm_gem_object_unreference_unlocked(obj);
   1482 	if (IS_ERR((void *)addr))
   1483 		return addr;
   1484 
   1485 	args->addr_ptr = (uint64_t) addr;
   1486 
   1487 	return 0;
   1488 }
   1489 
   1490 /**
   1491  * i915_gem_fault - fault a page into the GTT
   1492  * vma: VMA in question
   1493  * vmf: fault info
   1494  *
   1495  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
   1496  * from userspace.  The fault handler takes care of binding the object to
   1497  * the GTT (if needed), allocating and programming a fence register (again,
   1498  * only if needed based on whether the old reg is still valid or the object
   1499  * is tiled) and inserting a new PTE into the faulting process.
   1500  *
   1501  * Note that the faulting process may involve evicting existing objects
   1502  * from the GTT and/or fence registers to make room.  So performance may
   1503  * suffer if the GTT working set is large or there are few fence registers
   1504  * left.
   1505  */
   1506 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
   1507 {
   1508 	struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
   1509 	struct drm_device *dev = obj->base.dev;
   1510 	struct drm_i915_private *dev_priv = dev->dev_private;
   1511 	pgoff_t page_offset;
   1512 	unsigned long pfn;
   1513 	int ret = 0;
   1514 	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
   1515 
   1516 	intel_runtime_pm_get(dev_priv);
   1517 
   1518 	/* We don't use vmf->pgoff since that has the fake offset */
   1519 	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
   1520 		PAGE_SHIFT;
   1521 
   1522 	ret = i915_mutex_lock_interruptible(dev);
   1523 	if (ret)
   1524 		goto out;
   1525 
   1526 	trace_i915_gem_object_fault(obj, page_offset, true, write);
   1527 
   1528 	/* Try to flush the object off the GPU first without holding the lock.
   1529 	 * Upon reacquiring the lock, we will perform our sanity checks and then
   1530 	 * repeat the flush holding the lock in the normal manner to catch cases
   1531 	 * where we are gazumped.
   1532 	 */
   1533 	ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
   1534 	if (ret)
   1535 		goto unlock;
   1536 
   1537 	/* Access to snoopable pages through the GTT is incoherent. */
   1538 	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
   1539 		ret = -EINVAL;
   1540 		goto unlock;
   1541 	}
   1542 
   1543 	/* Now bind it into the GTT if needed */
   1544 	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE);
   1545 	if (ret)
   1546 		goto unlock;
   1547 
   1548 	ret = i915_gem_object_set_to_gtt_domain(obj, write);
   1549 	if (ret)
   1550 		goto unpin;
   1551 
   1552 	ret = i915_gem_object_get_fence(obj);
   1553 	if (ret)
   1554 		goto unpin;
   1555 
   1556 	obj->fault_mappable = true;
   1557 
   1558 	pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
   1559 	pfn >>= PAGE_SHIFT;
   1560 	pfn += page_offset;
   1561 
   1562 	/* Finally, remap it using the new GTT offset */
   1563 	ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
   1564 unpin:
   1565 	i915_gem_object_ggtt_unpin(obj);
   1566 unlock:
   1567 	mutex_unlock(&dev->struct_mutex);
   1568 out:
   1569 	switch (ret) {
   1570 	case -EIO:
   1571 		/* If this -EIO is due to a gpu hang, give the reset code a
   1572 		 * chance to clean up the mess. Otherwise return the proper
   1573 		 * SIGBUS. */
   1574 		if (i915_terminally_wedged(&dev_priv->gpu_error)) {
   1575 			ret = VM_FAULT_SIGBUS;
   1576 			break;
   1577 		}
   1578 	case -EAGAIN:
   1579 		/*
   1580 		 * EAGAIN means the gpu is hung and we'll wait for the error
   1581 		 * handler to reset everything when re-faulting in
   1582 		 * i915_mutex_lock_interruptible.
   1583 		 */
   1584 	case 0:
   1585 	case -ERESTARTSYS:
   1586 	case -EINTR:
   1587 	case -EBUSY:
   1588 		/*
   1589 		 * EBUSY is ok: this just means that another thread
   1590 		 * already did the job.
   1591 		 */
   1592 		ret = VM_FAULT_NOPAGE;
   1593 		break;
   1594 	case -ENOMEM:
   1595 		ret = VM_FAULT_OOM;
   1596 		break;
   1597 	case -ENOSPC:
   1598 	case -EFAULT:
   1599 		ret = VM_FAULT_SIGBUS;
   1600 		break;
   1601 	default:
   1602 		WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
   1603 		ret = VM_FAULT_SIGBUS;
   1604 		break;
   1605 	}
   1606 
   1607 	intel_runtime_pm_put(dev_priv);
   1608 	return ret;
   1609 }
   1610 
   1611 void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
   1612 {
   1613 	struct i915_vma *vma;
   1614 
   1615 	/*
   1616 	 * Only the global gtt is relevant for gtt memory mappings, so restrict
   1617 	 * list traversal to objects bound into the global address space. Note
   1618 	 * that the active list should be empty, but better safe than sorry.
   1619 	 */
   1620 	WARN_ON(!list_empty(&dev_priv->gtt.base.active_list));
   1621 	list_for_each_entry(vma, &dev_priv->gtt.base.active_list, mm_list)
   1622 		i915_gem_release_mmap(vma->obj);
   1623 	list_for_each_entry(vma, &dev_priv->gtt.base.inactive_list, mm_list)
   1624 		i915_gem_release_mmap(vma->obj);
   1625 }
   1626 
   1627 /**
   1628  * i915_gem_release_mmap - remove physical page mappings
   1629  * @obj: obj in question
   1630  *
   1631  * Preserve the reservation of the mmapping with the DRM core code, but
   1632  * relinquish ownership of the pages back to the system.
   1633  *
   1634  * It is vital that we remove the page mapping if we have mapped a tiled
   1635  * object through the GTT and then lose the fence register due to
   1636  * resource pressure. Similarly if the object has been moved out of the
   1637  * aperture, than pages mapped into userspace must be revoked. Removing the
   1638  * mapping will then trigger a page fault on the next user access, allowing
   1639  * fixup by i915_gem_fault().
   1640  */
   1641 void
   1642 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
   1643 {
   1644 	if (!obj->fault_mappable)
   1645 		return;
   1646 
   1647 	drm_vma_node_unmap(&obj->base.vma_node,
   1648 			   obj->base.dev->anon_inode->i_mapping);
   1649 	obj->fault_mappable = false;
   1650 }
   1651 
   1652 uint32_t
   1653 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
   1654 {
   1655 	uint32_t gtt_size;
   1656 
   1657 	if (INTEL_INFO(dev)->gen >= 4 ||
   1658 	    tiling_mode == I915_TILING_NONE)
   1659 		return size;
   1660 
   1661 	/* Previous chips need a power-of-two fence region when tiling */
   1662 	if (INTEL_INFO(dev)->gen == 3)
   1663 		gtt_size = 1024*1024;
   1664 	else
   1665 		gtt_size = 512*1024;
   1666 
   1667 	while (gtt_size < size)
   1668 		gtt_size <<= 1;
   1669 
   1670 	return gtt_size;
   1671 }
   1672 
   1673 /**
   1674  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
   1675  * @obj: object to check
   1676  *
   1677  * Return the required GTT alignment for an object, taking into account
   1678  * potential fence register mapping.
   1679  */
   1680 uint32_t
   1681 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
   1682 			   int tiling_mode, bool fenced)
   1683 {
   1684 	/*
   1685 	 * Minimum alignment is 4k (GTT page size), but might be greater
   1686 	 * if a fence register is needed for the object.
   1687 	 */
   1688 	if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
   1689 	    tiling_mode == I915_TILING_NONE)
   1690 		return 4096;
   1691 
   1692 	/*
   1693 	 * Previous chips need to be aligned to the size of the smallest
   1694 	 * fence register that can contain the object.
   1695 	 */
   1696 	return i915_gem_get_gtt_size(dev, size, tiling_mode);
   1697 }
   1698 
   1699 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
   1700 {
   1701 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
   1702 	int ret;
   1703 
   1704 	if (drm_vma_node_has_offset(&obj->base.vma_node))
   1705 		return 0;
   1706 
   1707 	dev_priv->mm.shrinker_no_lock_stealing = true;
   1708 
   1709 	ret = drm_gem_create_mmap_offset(&obj->base);
   1710 	if (ret != -ENOSPC)
   1711 		goto out;
   1712 
   1713 	/* Badly fragmented mmap space? The only way we can recover
   1714 	 * space is by destroying unwanted objects. We can't randomly release
   1715 	 * mmap_offsets as userspace expects them to be persistent for the
   1716 	 * lifetime of the objects. The closest we can is to release the
   1717 	 * offsets on purgeable objects by truncating it and marking it purged,
   1718 	 * which prevents userspace from ever using that object again.
   1719 	 */
   1720 	i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
   1721 	ret = drm_gem_create_mmap_offset(&obj->base);
   1722 	if (ret != -ENOSPC)
   1723 		goto out;
   1724 
   1725 	i915_gem_shrink_all(dev_priv);
   1726 	ret = drm_gem_create_mmap_offset(&obj->base);
   1727 out:
   1728 	dev_priv->mm.shrinker_no_lock_stealing = false;
   1729 
   1730 	return ret;
   1731 }
   1732 
   1733 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
   1734 {
   1735 	drm_gem_free_mmap_offset(&obj->base);
   1736 }
   1737 
   1738 int
   1739 i915_gem_mmap_gtt(struct drm_file *file,
   1740 		  struct drm_device *dev,
   1741 		  uint32_t handle,
   1742 		  uint64_t *offset)
   1743 {
   1744 	struct drm_i915_private *dev_priv = dev->dev_private;
   1745 	struct drm_i915_gem_object *obj;
   1746 	int ret;
   1747 
   1748 	ret = i915_mutex_lock_interruptible(dev);
   1749 	if (ret)
   1750 		return ret;
   1751 
   1752 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
   1753 	if (&obj->base == NULL) {
   1754 		ret = -ENOENT;
   1755 		goto unlock;
   1756 	}
   1757 
   1758 	if (obj->base.size > dev_priv->gtt.mappable_end) {
   1759 		ret = -E2BIG;
   1760 		goto out;
   1761 	}
   1762 
   1763 	if (obj->madv != I915_MADV_WILLNEED) {
   1764 		DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
   1765 		ret = -EFAULT;
   1766 		goto out;
   1767 	}
   1768 
   1769 	ret = i915_gem_object_create_mmap_offset(obj);
   1770 	if (ret)
   1771 		goto out;
   1772 
   1773 	*offset = drm_vma_node_offset_addr(&obj->base.vma_node);
   1774 
   1775 out:
   1776 	drm_gem_object_unreference(&obj->base);
   1777 unlock:
   1778 	mutex_unlock(&dev->struct_mutex);
   1779 	return ret;
   1780 }
   1781 
   1782 /**
   1783  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
   1784  * @dev: DRM device
   1785  * @data: GTT mapping ioctl data
   1786  * @file: GEM object info
   1787  *
   1788  * Simply returns the fake offset to userspace so it can mmap it.
   1789  * The mmap call will end up in drm_gem_mmap(), which will set things
   1790  * up so we can get faults in the handler above.
   1791  *
   1792  * The fault handler will take care of binding the object into the GTT
   1793  * (since it may have been evicted to make room for something), allocating
   1794  * a fence register, and mapping the appropriate aperture address into
   1795  * userspace.
   1796  */
   1797 int
   1798 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
   1799 			struct drm_file *file)
   1800 {
   1801 	struct drm_i915_gem_mmap_gtt *args = data;
   1802 
   1803 	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
   1804 }
   1805 
   1806 /* Immediately discard the backing storage */
   1807 static void
   1808 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
   1809 {
   1810 	struct inode *inode;
   1811 
   1812 	i915_gem_object_free_mmap_offset(obj);
   1813 
   1814 	if (obj->base.filp == NULL)
   1815 		return;
   1816 
   1817 	/* Our goal here is to return as much of the memory as
   1818 	 * is possible back to the system as we are called from OOM.
   1819 	 * To do this we must instruct the shmfs to drop all of its
   1820 	 * backing pages, *now*.
   1821 	 */
   1822 	inode = file_inode(obj->base.filp);
   1823 	shmem_truncate_range(inode, 0, (loff_t)-1);
   1824 
   1825 	obj->madv = __I915_MADV_PURGED;
   1826 }
   1827 
   1828 static inline int
   1829 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
   1830 {
   1831 	return obj->madv == I915_MADV_DONTNEED;
   1832 }
   1833 
   1834 static void
   1835 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
   1836 {
   1837 	struct sg_page_iter sg_iter;
   1838 	int ret;
   1839 
   1840 	BUG_ON(obj->madv == __I915_MADV_PURGED);
   1841 
   1842 	ret = i915_gem_object_set_to_cpu_domain(obj, true);
   1843 	if (ret) {
   1844 		/* In the event of a disaster, abandon all caches and
   1845 		 * hope for the best.
   1846 		 */
   1847 		WARN_ON(ret != -EIO);
   1848 		i915_gem_clflush_object(obj, true);
   1849 		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
   1850 	}
   1851 
   1852 	if (i915_gem_object_needs_bit17_swizzle(obj))
   1853 		i915_gem_object_save_bit_17_swizzle(obj);
   1854 
   1855 	if (obj->madv == I915_MADV_DONTNEED)
   1856 		obj->dirty = 0;
   1857 
   1858 	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
   1859 		struct page *page = sg_page_iter_page(&sg_iter);
   1860 
   1861 		if (obj->dirty)
   1862 			set_page_dirty(page);
   1863 
   1864 		if (obj->madv == I915_MADV_WILLNEED)
   1865 			mark_page_accessed(page);
   1866 
   1867 		page_cache_release(page);
   1868 	}
   1869 	obj->dirty = 0;
   1870 
   1871 	sg_free_table(obj->pages);
   1872 	kfree(obj->pages);
   1873 }
   1874 
   1875 int
   1876 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
   1877 {
   1878 	const struct drm_i915_gem_object_ops *ops = obj->ops;
   1879 
   1880 	if (obj->pages == NULL)
   1881 		return 0;
   1882 
   1883 	if (obj->pages_pin_count)
   1884 		return -EBUSY;
   1885 
   1886 	BUG_ON(i915_gem_obj_bound_any(obj));
   1887 
   1888 	/* ->put_pages might need to allocate memory for the bit17 swizzle
   1889 	 * array, hence protect them from being reaped by removing them from gtt
   1890 	 * lists early. */
   1891 	list_del(&obj->global_list);
   1892 
   1893 	ops->put_pages(obj);
   1894 	obj->pages = NULL;
   1895 
   1896 	if (i915_gem_object_is_purgeable(obj))
   1897 		i915_gem_object_truncate(obj);
   1898 
   1899 	return 0;
   1900 }
   1901 
   1902 static unsigned long
   1903 __i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
   1904 		  bool purgeable_only)
   1905 {
   1906 	struct list_head still_bound_list;
   1907 	struct drm_i915_gem_object *obj, *next;
   1908 	unsigned long count = 0;
   1909 
   1910 	list_for_each_entry_safe(obj, next,
   1911 				 &dev_priv->mm.unbound_list,
   1912 				 global_list) {
   1913 		if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
   1914 		    i915_gem_object_put_pages(obj) == 0) {
   1915 			count += obj->base.size >> PAGE_SHIFT;
   1916 			if (count >= target)
   1917 				return count;
   1918 		}
   1919 	}
   1920 
   1921 	/*
   1922 	 * As we may completely rewrite the bound list whilst unbinding
   1923 	 * (due to retiring requests) we have to strictly process only
   1924 	 * one element of the list at the time, and recheck the list
   1925 	 * on every iteration.
   1926 	 */
   1927 	INIT_LIST_HEAD(&still_bound_list);
   1928 	while (count < target && !list_empty(&dev_priv->mm.bound_list)) {
   1929 		struct i915_vma *vma, *v;
   1930 
   1931 		obj = list_first_entry(&dev_priv->mm.bound_list,
   1932 				       typeof(*obj), global_list);
   1933 		list_move_tail(&obj->global_list, &still_bound_list);
   1934 
   1935 		if (!i915_gem_object_is_purgeable(obj) && purgeable_only)
   1936 			continue;
   1937 
   1938 		/*
   1939 		 * Hold a reference whilst we unbind this object, as we may
   1940 		 * end up waiting for and retiring requests. This might
   1941 		 * release the final reference (held by the active list)
   1942 		 * and result in the object being freed from under us.
   1943 		 * in this object being freed.
   1944 		 *
   1945 		 * Note 1: Shrinking the bound list is special since only active
   1946 		 * (and hence bound objects) can contain such limbo objects, so
   1947 		 * we don't need special tricks for shrinking the unbound list.
   1948 		 * The only other place where we have to be careful with active
   1949 		 * objects suddenly disappearing due to retiring requests is the
   1950 		 * eviction code.
   1951 		 *
   1952 		 * Note 2: Even though the bound list doesn't hold a reference
   1953 		 * to the object we can safely grab one here: The final object
   1954 		 * unreferencing and the bound_list are both protected by the
   1955 		 * dev->struct_mutex and so we won't ever be able to observe an
   1956 		 * object on the bound_list with a reference count equals 0.
   1957 		 */
   1958 		drm_gem_object_reference(&obj->base);
   1959 
   1960 		list_for_each_entry_safe(vma, v, &obj->vma_list, vma_link)
   1961 			if (i915_vma_unbind(vma))
   1962 				break;
   1963 
   1964 		if (i915_gem_object_put_pages(obj) == 0)
   1965 			count += obj->base.size >> PAGE_SHIFT;
   1966 
   1967 		drm_gem_object_unreference(&obj->base);
   1968 	}
   1969 	list_splice(&still_bound_list, &dev_priv->mm.bound_list);
   1970 
   1971 	return count;
   1972 }
   1973 
   1974 static unsigned long
   1975 i915_gem_purge(struct drm_i915_private *dev_priv, long target)
   1976 {
   1977 	return __i915_gem_shrink(dev_priv, target, true);
   1978 }
   1979 
   1980 static unsigned long
   1981 i915_gem_shrink_all(struct drm_i915_private *dev_priv)
   1982 {
   1983 	struct drm_i915_gem_object *obj, *next;
   1984 	long freed = 0;
   1985 
   1986 	i915_gem_evict_everything(dev_priv->dev);
   1987 
   1988 	list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list,
   1989 				 global_list) {
   1990 		if (i915_gem_object_put_pages(obj) == 0)
   1991 			freed += obj->base.size >> PAGE_SHIFT;
   1992 	}
   1993 	return freed;
   1994 }
   1995 
   1996 static int
   1997 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
   1998 {
   1999 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
   2000 	int page_count, i;
   2001 	struct address_space *mapping;
   2002 	struct sg_table *st;
   2003 	struct scatterlist *sg;
   2004 	struct sg_page_iter sg_iter;
   2005 	struct page *page;
   2006 	unsigned long last_pfn = 0;	/* suppress gcc warning */
   2007 	gfp_t gfp;
   2008 
   2009 	/* Assert that the object is not currently in any GPU domain. As it
   2010 	 * wasn't in the GTT, there shouldn't be any way it could have been in
   2011 	 * a GPU cache
   2012 	 */
   2013 	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
   2014 	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
   2015 
   2016 	st = kmalloc(sizeof(*st), GFP_KERNEL);
   2017 	if (st == NULL)
   2018 		return -ENOMEM;
   2019 
   2020 	page_count = obj->base.size / PAGE_SIZE;
   2021 	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
   2022 		kfree(st);
   2023 		return -ENOMEM;
   2024 	}
   2025 
   2026 	/* Get the list of pages out of our struct file.  They'll be pinned
   2027 	 * at this point until we release them.
   2028 	 *
   2029 	 * Fail silently without starting the shrinker
   2030 	 */
   2031 	mapping = file_inode(obj->base.filp)->i_mapping;
   2032 	gfp = mapping_gfp_mask(mapping);
   2033 	gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
   2034 	gfp &= ~(__GFP_IO | __GFP_WAIT);
   2035 	sg = st->sgl;
   2036 	st->nents = 0;
   2037 	for (i = 0; i < page_count; i++) {
   2038 		page = shmem_read_mapping_page_gfp(mapping, i, gfp);
   2039 		if (IS_ERR(page)) {
   2040 			i915_gem_purge(dev_priv, page_count);
   2041 			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
   2042 		}
   2043 		if (IS_ERR(page)) {
   2044 			/* We've tried hard to allocate the memory by reaping
   2045 			 * our own buffer, now let the real VM do its job and
   2046 			 * go down in flames if truly OOM.
   2047 			 */
   2048 			gfp &= ~(__GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD);
   2049 			gfp |= __GFP_IO | __GFP_WAIT;
   2050 
   2051 			i915_gem_shrink_all(dev_priv);
   2052 			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
   2053 			if (IS_ERR(page))
   2054 				goto err_pages;
   2055 
   2056 			gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
   2057 			gfp &= ~(__GFP_IO | __GFP_WAIT);
   2058 		}
   2059 #ifdef CONFIG_SWIOTLB
   2060 		if (swiotlb_nr_tbl()) {
   2061 			st->nents++;
   2062 			sg_set_page(sg, page, PAGE_SIZE, 0);
   2063 			sg = sg_next(sg);
   2064 			continue;
   2065 		}
   2066 #endif
   2067 		if (!i || page_to_pfn(page) != last_pfn + 1) {
   2068 			if (i)
   2069 				sg = sg_next(sg);
   2070 			st->nents++;
   2071 			sg_set_page(sg, page, PAGE_SIZE, 0);
   2072 		} else {
   2073 			sg->length += PAGE_SIZE;
   2074 		}
   2075 		last_pfn = page_to_pfn(page);
   2076 
   2077 		/* Check that the i965g/gm workaround works. */
   2078 		WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
   2079 	}
   2080 #ifdef CONFIG_SWIOTLB
   2081 	if (!swiotlb_nr_tbl())
   2082 #endif
   2083 		sg_mark_end(sg);
   2084 	obj->pages = st;
   2085 
   2086 	if (i915_gem_object_needs_bit17_swizzle(obj))
   2087 		i915_gem_object_do_bit_17_swizzle(obj);
   2088 
   2089 	return 0;
   2090 
   2091 err_pages:
   2092 	sg_mark_end(sg);
   2093 	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
   2094 		page_cache_release(sg_page_iter_page(&sg_iter));
   2095 	sg_free_table(st);
   2096 	kfree(st);
   2097 	return PTR_ERR(page);
   2098 }
   2099 
   2100 /* Ensure that the associated pages are gathered from the backing storage
   2101  * and pinned into our object. i915_gem_object_get_pages() may be called
   2102  * multiple times before they are released by a single call to
   2103  * i915_gem_object_put_pages() - once the pages are no longer referenced
   2104  * either as a result of memory pressure (reaping pages under the shrinker)
   2105  * or as the object is itself released.
   2106  */
   2107 int
   2108 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
   2109 {
   2110 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
   2111 	const struct drm_i915_gem_object_ops *ops = obj->ops;
   2112 	int ret;
   2113 
   2114 	if (obj->pages)
   2115 		return 0;
   2116 
   2117 	if (obj->madv != I915_MADV_WILLNEED) {
   2118 		DRM_DEBUG("Attempting to obtain a purgeable object\n");
   2119 		return -EFAULT;
   2120 	}
   2121 
   2122 	BUG_ON(obj->pages_pin_count);
   2123 
   2124 	ret = ops->get_pages(obj);
   2125 	if (ret)
   2126 		return ret;
   2127 
   2128 	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
   2129 	return 0;
   2130 }
   2131 
   2132 static void
   2133 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
   2134 			       struct intel_ring_buffer *ring)
   2135 {
   2136 	struct drm_device *dev = obj->base.dev;
   2137 	struct drm_i915_private *dev_priv = dev->dev_private;
   2138 	u32 seqno = intel_ring_get_seqno(ring);
   2139 
   2140 	BUG_ON(ring == NULL);
   2141 	if (obj->ring != ring && obj->last_write_seqno) {
   2142 		/* Keep the seqno relative to the current ring */
   2143 		obj->last_write_seqno = seqno;
   2144 	}
   2145 	obj->ring = ring;
   2146 
   2147 	/* Add a reference if we're newly entering the active list. */
   2148 	if (!obj->active) {
   2149 		drm_gem_object_reference(&obj->base);
   2150 		obj->active = 1;
   2151 	}
   2152 
   2153 	list_move_tail(&obj->ring_list, &ring->active_list);
   2154 
   2155 	obj->last_read_seqno = seqno;
   2156 
   2157 	if (obj->fenced_gpu_access) {
   2158 		obj->last_fenced_seqno = seqno;
   2159 
   2160 		/* Bump MRU to take account of the delayed flush */
   2161 		if (obj->fence_reg != I915_FENCE_REG_NONE) {
   2162 			struct drm_i915_fence_reg *reg;
   2163 
   2164 			reg = &dev_priv->fence_regs[obj->fence_reg];
   2165 			list_move_tail(&reg->lru_list,
   2166 				       &dev_priv->mm.fence_list);
   2167 		}
   2168 	}
   2169 }
   2170 
   2171 void i915_vma_move_to_active(struct i915_vma *vma,
   2172 			     struct intel_ring_buffer *ring)
   2173 {
   2174 	list_move_tail(&vma->mm_list, &vma->vm->active_list);
   2175 	return i915_gem_object_move_to_active(vma->obj, ring);
   2176 }
   2177 
   2178 static void
   2179 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
   2180 {
   2181 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
   2182 	struct i915_address_space *vm;
   2183 	struct i915_vma *vma;
   2184 
   2185 	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
   2186 	BUG_ON(!obj->active);
   2187 
   2188 	list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
   2189 		vma = i915_gem_obj_to_vma(obj, vm);
   2190 		if (vma && !list_empty(&vma->mm_list))
   2191 			list_move_tail(&vma->mm_list, &vm->inactive_list);
   2192 	}
   2193 
   2194 	list_del_init(&obj->ring_list);
   2195 	obj->ring = NULL;
   2196 
   2197 	obj->last_read_seqno = 0;
   2198 	obj->last_write_seqno = 0;
   2199 	obj->base.write_domain = 0;
   2200 
   2201 	obj->last_fenced_seqno = 0;
   2202 	obj->fenced_gpu_access = false;
   2203 
   2204 	obj->active = 0;
   2205 	drm_gem_object_unreference(&obj->base);
   2206 
   2207 	WARN_ON(i915_verify_lists(dev));
   2208 }
   2209 
   2210 static int
   2211 i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
   2212 {
   2213 	struct drm_i915_private *dev_priv = dev->dev_private;
   2214 	struct intel_ring_buffer *ring;
   2215 	int ret, i, j;
   2216 
   2217 	/* Carefully retire all requests without writing to the rings */
   2218 	for_each_ring(ring, dev_priv, i) {
   2219 		ret = intel_ring_idle(ring);
   2220 		if (ret)
   2221 			return ret;
   2222 	}
   2223 	i915_gem_retire_requests(dev);
   2224 
   2225 	/* Finally reset hw state */
   2226 	for_each_ring(ring, dev_priv, i) {
   2227 		intel_ring_init_seqno(ring, seqno);
   2228 
   2229 		for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
   2230 			ring->sync_seqno[j] = 0;
   2231 	}
   2232 
   2233 	return 0;
   2234 }
   2235 
   2236 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
   2237 {
   2238 	struct drm_i915_private *dev_priv = dev->dev_private;
   2239 	int ret;
   2240 
   2241 	if (seqno == 0)
   2242 		return -EINVAL;
   2243 
   2244 	/* HWS page needs to be set less than what we
   2245 	 * will inject to ring
   2246 	 */
   2247 	ret = i915_gem_init_seqno(dev, seqno - 1);
   2248 	if (ret)
   2249 		return ret;
   2250 
   2251 	/* Carefully set the last_seqno value so that wrap
   2252 	 * detection still works
   2253 	 */
   2254 	dev_priv->next_seqno = seqno;
   2255 	dev_priv->last_seqno = seqno - 1;
   2256 	if (dev_priv->last_seqno == 0)
   2257 		dev_priv->last_seqno--;
   2258 
   2259 	return 0;
   2260 }
   2261 
   2262 int
   2263 i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
   2264 {
   2265 	struct drm_i915_private *dev_priv = dev->dev_private;
   2266 
   2267 	/* reserve 0 for non-seqno */
   2268 	if (dev_priv->next_seqno == 0) {
   2269 		int ret = i915_gem_init_seqno(dev, 0);
   2270 		if (ret)
   2271 			return ret;
   2272 
   2273 		dev_priv->next_seqno = 1;
   2274 	}
   2275 
   2276 	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
   2277 	return 0;
   2278 }
   2279 
   2280 int __i915_add_request(struct intel_ring_buffer *ring,
   2281 		       struct drm_file *file,
   2282 		       struct drm_i915_gem_object *obj,
   2283 		       u32 *out_seqno)
   2284 {
   2285 	struct drm_i915_private *dev_priv = ring->dev->dev_private;
   2286 	struct drm_i915_gem_request *request;
   2287 	u32 request_ring_position, request_start;
   2288 	int ret;
   2289 
   2290 	request_start = intel_ring_get_tail(ring);
   2291 	/*
   2292 	 * Emit any outstanding flushes - execbuf can fail to emit the flush
   2293 	 * after having emitted the batchbuffer command. Hence we need to fix
   2294 	 * things up similar to emitting the lazy request. The difference here
   2295 	 * is that the flush _must_ happen before the next request, no matter
   2296 	 * what.
   2297 	 */
   2298 	ret = intel_ring_flush_all_caches(ring);
   2299 	if (ret)
   2300 		return ret;
   2301 
   2302 	request = ring->preallocated_lazy_request;
   2303 	if (WARN_ON(request == NULL))
   2304 		return -ENOMEM;
   2305 
   2306 	/* Record the position of the start of the request so that
   2307 	 * should we detect the updated seqno part-way through the
   2308 	 * GPU processing the request, we never over-estimate the
   2309 	 * position of the head.
   2310 	 */
   2311 	request_ring_position = intel_ring_get_tail(ring);
   2312 
   2313 	ret = ring->add_request(ring);
   2314 	if (ret)
   2315 		return ret;
   2316 
   2317 	request->seqno = intel_ring_get_seqno(ring);
   2318 	request->ring = ring;
   2319 	request->head = request_start;
   2320 	request->tail = request_ring_position;
   2321 
   2322 	/* Whilst this request exists, batch_obj will be on the
   2323 	 * active_list, and so will hold the active reference. Only when this
   2324 	 * request is retired will the the batch_obj be moved onto the
   2325 	 * inactive_list and lose its active reference. Hence we do not need
   2326 	 * to explicitly hold another reference here.
   2327 	 */
   2328 	request->batch_obj = obj;
   2329 
   2330 	/* Hold a reference to the current context so that we can inspect
   2331 	 * it later in case a hangcheck error event fires.
   2332 	 */
   2333 	request->ctx = ring->last_context;
   2334 	if (request->ctx)
   2335 		i915_gem_context_reference(request->ctx);
   2336 
   2337 	request->emitted_jiffies = jiffies;
   2338 	list_add_tail(&request->list, &ring->request_list);
   2339 	request->file_priv = NULL;
   2340 
   2341 	if (file) {
   2342 		struct drm_i915_file_private *file_priv = file->driver_priv;
   2343 
   2344 		spin_lock(&file_priv->mm.lock);
   2345 		request->file_priv = file_priv;
   2346 		list_add_tail(&request->client_list,
   2347 			      &file_priv->mm.request_list);
   2348 		spin_unlock(&file_priv->mm.lock);
   2349 	}
   2350 
   2351 	trace_i915_gem_request_add(ring, request->seqno);
   2352 	ring->outstanding_lazy_seqno = 0;
   2353 	ring->preallocated_lazy_request = NULL;
   2354 
   2355 	if (!dev_priv->ums.mm_suspended) {
   2356 		i915_queue_hangcheck(ring->dev);
   2357 
   2358 		cancel_delayed_work_sync(&dev_priv->mm.idle_work);
   2359 		queue_delayed_work(dev_priv->wq,
   2360 				   &dev_priv->mm.retire_work,
   2361 				   round_jiffies_up_relative(HZ));
   2362 		intel_mark_busy(dev_priv->dev);
   2363 	}
   2364 
   2365 	if (out_seqno)
   2366 		*out_seqno = request->seqno;
   2367 	return 0;
   2368 }
   2369 
   2370 static inline void
   2371 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
   2372 {
   2373 	struct drm_i915_file_private *file_priv = request->file_priv;
   2374 
   2375 	if (!file_priv)
   2376 		return;
   2377 
   2378 	spin_lock(&file_priv->mm.lock);
   2379 	list_del(&request->client_list);
   2380 	request->file_priv = NULL;
   2381 	spin_unlock(&file_priv->mm.lock);
   2382 }
   2383 
   2384 static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
   2385 				   const struct i915_hw_context *ctx)
   2386 {
   2387 	unsigned long elapsed;
   2388 
   2389 	elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
   2390 
   2391 	if (ctx->hang_stats.banned)
   2392 		return true;
   2393 
   2394 	if (elapsed <= DRM_I915_CTX_BAN_PERIOD) {
   2395 		if (!i915_gem_context_is_default(ctx)) {
   2396 			DRM_DEBUG("context hanging too fast, banning!\n");
   2397 			return true;
   2398 		} else if (dev_priv->gpu_error.stop_rings == 0) {
   2399 			DRM_ERROR("gpu hanging too fast, banning!\n");
   2400 			return true;
   2401 		}
   2402 	}
   2403 
   2404 	return false;
   2405 }
   2406 
   2407 static void i915_set_reset_status(struct drm_i915_private *dev_priv,
   2408 				  struct i915_hw_context *ctx,
   2409 				  const bool guilty)
   2410 {
   2411 	struct i915_ctx_hang_stats *hs;
   2412 
   2413 	if (WARN_ON(!ctx))
   2414 		return;
   2415 
   2416 	hs = &ctx->hang_stats;
   2417 
   2418 	if (guilty) {
   2419 		hs->banned = i915_context_is_banned(dev_priv, ctx);
   2420 		hs->batch_active++;
   2421 		hs->guilty_ts = get_seconds();
   2422 	} else {
   2423 		hs->batch_pending++;
   2424 	}
   2425 }
   2426 
   2427 static void i915_gem_free_request(struct drm_i915_gem_request *request)
   2428 {
   2429 	list_del(&request->list);
   2430 	i915_gem_request_remove_from_client(request);
   2431 
   2432 	if (request->ctx)
   2433 		i915_gem_context_unreference(request->ctx);
   2434 
   2435 	kfree(request);
   2436 }
   2437 
   2438 struct drm_i915_gem_request *
   2439 i915_gem_find_active_request(struct intel_ring_buffer *ring)
   2440 {
   2441 	struct drm_i915_gem_request *request;
   2442 	u32 completed_seqno;
   2443 
   2444 	completed_seqno = ring->get_seqno(ring, false);
   2445 
   2446 	list_for_each_entry(request, &ring->request_list, list) {
   2447 		if (i915_seqno_passed(completed_seqno, request->seqno))
   2448 			continue;
   2449 
   2450 		return request;
   2451 	}
   2452 
   2453 	return NULL;
   2454 }
   2455 
   2456 static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
   2457 				       struct intel_ring_buffer *ring)
   2458 {
   2459 	struct drm_i915_gem_request *request;
   2460 	bool ring_hung;
   2461 
   2462 	request = i915_gem_find_active_request(ring);
   2463 
   2464 	if (request == NULL)
   2465 		return;
   2466 
   2467 	ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
   2468 
   2469 	i915_set_reset_status(dev_priv, request->ctx, ring_hung);
   2470 
   2471 	list_for_each_entry_continue(request, &ring->request_list, list)
   2472 		i915_set_reset_status(dev_priv, request->ctx, false);
   2473 }
   2474 
   2475 static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
   2476 					struct intel_ring_buffer *ring)
   2477 {
   2478 	while (!list_empty(&ring->active_list)) {
   2479 		struct drm_i915_gem_object *obj;
   2480 
   2481 		obj = list_first_entry(&ring->active_list,
   2482 				       struct drm_i915_gem_object,
   2483 				       ring_list);
   2484 
   2485 		i915_gem_object_move_to_inactive(obj);
   2486 	}
   2487 
   2488 	/*
   2489 	 * We must free the requests after all the corresponding objects have
   2490 	 * been moved off active lists. Which is the same order as the normal
   2491 	 * retire_requests function does. This is important if object hold
   2492 	 * implicit references on things like e.g. ppgtt address spaces through
   2493 	 * the request.
   2494 	 */
   2495 	while (!list_empty(&ring->request_list)) {
   2496 		struct drm_i915_gem_request *request;
   2497 
   2498 		request = list_first_entry(&ring->request_list,
   2499 					   struct drm_i915_gem_request,
   2500 					   list);
   2501 
   2502 		i915_gem_free_request(request);
   2503 	}
   2504 }
   2505 
   2506 void i915_gem_restore_fences(struct drm_device *dev)
   2507 {
   2508 	struct drm_i915_private *dev_priv = dev->dev_private;
   2509 	int i;
   2510 
   2511 	for (i = 0; i < dev_priv->num_fence_regs; i++) {
   2512 		struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
   2513 
   2514 		/*
   2515 		 * Commit delayed tiling changes if we have an object still
   2516 		 * attached to the fence, otherwise just clear the fence.
   2517 		 */
   2518 		if (reg->obj) {
   2519 			i915_gem_object_update_fence(reg->obj, reg,
   2520 						     reg->obj->tiling_mode);
   2521 		} else {
   2522 			i915_gem_write_fence(dev, i, NULL);
   2523 		}
   2524 	}
   2525 }
   2526 
   2527 void i915_gem_reset(struct drm_device *dev)
   2528 {
   2529 	struct drm_i915_private *dev_priv = dev->dev_private;
   2530 	struct intel_ring_buffer *ring;
   2531 	int i;
   2532 
   2533 	/*
   2534 	 * Before we free the objects from the requests, we need to inspect
   2535 	 * them for finding the guilty party. As the requests only borrow
   2536 	 * their reference to the objects, the inspection must be done first.
   2537 	 */
   2538 	for_each_ring(ring, dev_priv, i)
   2539 		i915_gem_reset_ring_status(dev_priv, ring);
   2540 
   2541 	for_each_ring(ring, dev_priv, i)
   2542 		i915_gem_reset_ring_cleanup(dev_priv, ring);
   2543 
   2544 	i915_gem_cleanup_ringbuffer(dev);
   2545 
   2546 	i915_gem_context_reset(dev);
   2547 
   2548 	i915_gem_restore_fences(dev);
   2549 }
   2550 
   2551 /**
   2552  * This function clears the request list as sequence numbers are passed.
   2553  */
   2554 static void
   2555 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
   2556 {
   2557 	uint32_t seqno;
   2558 
   2559 	if (list_empty(&ring->request_list))
   2560 		return;
   2561 
   2562 	WARN_ON(i915_verify_lists(ring->dev));
   2563 
   2564 	seqno = ring->get_seqno(ring, true);
   2565 
   2566 	/* Move any buffers on the active list that are no longer referenced
   2567 	 * by the ringbuffer to the flushing/inactive lists as appropriate,
   2568 	 * before we free the context associated with the requests.
   2569 	 */
   2570 	while (!list_empty(&ring->active_list)) {
   2571 		struct drm_i915_gem_object *obj;
   2572 
   2573 		obj = list_first_entry(&ring->active_list,
   2574 				      struct drm_i915_gem_object,
   2575 				      ring_list);
   2576 
   2577 		if (!i915_seqno_passed(seqno, obj->last_read_seqno))
   2578 			break;
   2579 
   2580 		i915_gem_object_move_to_inactive(obj);
   2581 	}
   2582 
   2583 
   2584 	while (!list_empty(&ring->request_list)) {
   2585 		struct drm_i915_gem_request *request;
   2586 
   2587 		request = list_first_entry(&ring->request_list,
   2588 					   struct drm_i915_gem_request,
   2589 					   list);
   2590 
   2591 		if (!i915_seqno_passed(seqno, request->seqno))
   2592 			break;
   2593 
   2594 		trace_i915_gem_request_retire(ring, request->seqno);
   2595 		/* We know the GPU must have read the request to have
   2596 		 * sent us the seqno + interrupt, so use the position
   2597 		 * of tail of the request to update the last known position
   2598 		 * of the GPU head.
   2599 		 */
   2600 		ring->last_retired_head = request->tail;
   2601 
   2602 		i915_gem_free_request(request);
   2603 	}
   2604 
   2605 	if (unlikely(ring->trace_irq_seqno &&
   2606 		     i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
   2607 		ring->irq_put(ring);
   2608 		ring->trace_irq_seqno = 0;
   2609 	}
   2610 
   2611 	WARN_ON(i915_verify_lists(ring->dev));
   2612 }
   2613 
   2614 bool
   2615 i915_gem_retire_requests(struct drm_device *dev)
   2616 {
   2617 	struct drm_i915_private *dev_priv = dev->dev_private;
   2618 	struct intel_ring_buffer *ring;
   2619 	bool idle = true;
   2620 	int i;
   2621 
   2622 	for_each_ring(ring, dev_priv, i) {
   2623 		i915_gem_retire_requests_ring(ring);
   2624 		idle &= list_empty(&ring->request_list);
   2625 	}
   2626 
   2627 	if (idle)
   2628 		mod_delayed_work(dev_priv->wq,
   2629 				   &dev_priv->mm.idle_work,
   2630 				   msecs_to_jiffies(100));
   2631 
   2632 	return idle;
   2633 }
   2634 
   2635 static void
   2636 i915_gem_retire_work_handler(struct work_struct *work)
   2637 {
   2638 	struct drm_i915_private *dev_priv =
   2639 		container_of(work, typeof(*dev_priv), mm.retire_work.work);
   2640 	struct drm_device *dev = dev_priv->dev;
   2641 	bool idle;
   2642 
   2643 	/* Come back later if the device is busy... */
   2644 	idle = false;
   2645 	if (mutex_trylock(&dev->struct_mutex)) {
   2646 		idle = i915_gem_retire_requests(dev);
   2647 		mutex_unlock(&dev->struct_mutex);
   2648 	}
   2649 	if (!idle)
   2650 		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
   2651 				   round_jiffies_up_relative(HZ));
   2652 }
   2653 
   2654 static void
   2655 i915_gem_idle_work_handler(struct work_struct *work)
   2656 {
   2657 	struct drm_i915_private *dev_priv =
   2658 		container_of(work, typeof(*dev_priv), mm.idle_work.work);
   2659 
   2660 	intel_mark_idle(dev_priv->dev);
   2661 }
   2662 
   2663 /**
   2664  * Ensures that an object will eventually get non-busy by flushing any required
   2665  * write domains, emitting any outstanding lazy request and retiring and
   2666  * completed requests.
   2667  */
   2668 static int
   2669 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
   2670 {
   2671 	int ret;
   2672 
   2673 	if (obj->active) {
   2674 		ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
   2675 		if (ret)
   2676 			return ret;
   2677 
   2678 		i915_gem_retire_requests_ring(obj->ring);
   2679 	}
   2680 
   2681 	return 0;
   2682 }
   2683 
   2684 /**
   2685  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
   2686  * @DRM_IOCTL_ARGS: standard ioctl arguments
   2687  *
   2688  * Returns 0 if successful, else an error is returned with the remaining time in
   2689  * the timeout parameter.
   2690  *  -ETIME: object is still busy after timeout
   2691  *  -ERESTARTSYS: signal interrupted the wait
   2692  *  -ENONENT: object doesn't exist
   2693  * Also possible, but rare:
   2694  *  -EAGAIN: GPU wedged
   2695  *  -ENOMEM: damn
   2696  *  -ENODEV: Internal IRQ fail
   2697  *  -E?: The add request failed
   2698  *
   2699  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
   2700  * non-zero timeout parameter the wait ioctl will wait for the given number of
   2701  * nanoseconds on an object becoming unbusy. Since the wait itself does so
   2702  * without holding struct_mutex the object may become re-busied before this
   2703  * function completes. A similar but shorter * race condition exists in the busy
   2704  * ioctl
   2705  */
   2706 int
   2707 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
   2708 {
   2709 	struct drm_i915_private *dev_priv = dev->dev_private;
   2710 	struct drm_i915_gem_wait *args = data;
   2711 	struct drm_i915_gem_object *obj;
   2712 	struct intel_ring_buffer *ring = NULL;
   2713 	struct timespec timeout_stack, *timeout = NULL;
   2714 	unsigned reset_counter;
   2715 	u32 seqno = 0;
   2716 	int ret = 0;
   2717 
   2718 	if (args->timeout_ns >= 0) {
   2719 		timeout_stack = ns_to_timespec(args->timeout_ns);
   2720 		timeout = &timeout_stack;
   2721 	}
   2722 
   2723 	ret = i915_mutex_lock_interruptible(dev);
   2724 	if (ret)
   2725 		return ret;
   2726 
   2727 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
   2728 	if (&obj->base == NULL) {
   2729 		mutex_unlock(&dev->struct_mutex);
   2730 		return -ENOENT;
   2731 	}
   2732 
   2733 	/* Need to make sure the object gets inactive eventually. */
   2734 	ret = i915_gem_object_flush_active(obj);
   2735 	if (ret)
   2736 		goto out;
   2737 
   2738 	if (obj->active) {
   2739 		seqno = obj->last_read_seqno;
   2740 		ring = obj->ring;
   2741 	}
   2742 
   2743 	if (seqno == 0)
   2744 		 goto out;
   2745 
   2746 	/* Do this after OLR check to make sure we make forward progress polling
   2747 	 * on this IOCTL with a 0 timeout (like busy ioctl)
   2748 	 */
   2749 	if (!args->timeout_ns) {
   2750 		ret = -ETIME;
   2751 		goto out;
   2752 	}
   2753 
   2754 	drm_gem_object_unreference(&obj->base);
   2755 	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
   2756 	mutex_unlock(&dev->struct_mutex);
   2757 
   2758 	ret = __wait_seqno(ring, seqno, reset_counter, true, timeout, file->driver_priv);
   2759 	if (timeout)
   2760 		args->timeout_ns = timespec_to_ns(timeout);
   2761 	return ret;
   2762 
   2763 out:
   2764 	drm_gem_object_unreference(&obj->base);
   2765 	mutex_unlock(&dev->struct_mutex);
   2766 	return ret;
   2767 }
   2768 
   2769 /**
   2770  * i915_gem_object_sync - sync an object to a ring.
   2771  *
   2772  * @obj: object which may be in use on another ring.
   2773  * @to: ring we wish to use the object on. May be NULL.
   2774  *
   2775  * This code is meant to abstract object synchronization with the GPU.
   2776  * Calling with NULL implies synchronizing the object with the CPU
   2777  * rather than a particular GPU ring.
   2778  *
   2779  * Returns 0 if successful, else propagates up the lower layer error.
   2780  */
   2781 int
   2782 i915_gem_object_sync(struct drm_i915_gem_object *obj,
   2783 		     struct intel_ring_buffer *to)
   2784 {
   2785 	struct intel_ring_buffer *from = obj->ring;
   2786 	u32 seqno;
   2787 	int ret, idx;
   2788 
   2789 	if (from == NULL || to == from)
   2790 		return 0;
   2791 
   2792 	if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
   2793 		return i915_gem_object_wait_rendering(obj, false);
   2794 
   2795 	idx = intel_ring_sync_index(from, to);
   2796 
   2797 	seqno = obj->last_read_seqno;
   2798 	if (seqno <= from->sync_seqno[idx])
   2799 		return 0;
   2800 
   2801 	ret = i915_gem_check_olr(obj->ring, seqno);
   2802 	if (ret)
   2803 		return ret;
   2804 
   2805 	trace_i915_gem_ring_sync_to(from, to, seqno);
   2806 	ret = to->sync_to(to, from, seqno);
   2807 	if (!ret)
   2808 		/* We use last_read_seqno because sync_to()
   2809 		 * might have just caused seqno wrap under
   2810 		 * the radar.
   2811 		 */
   2812 		from->sync_seqno[idx] = obj->last_read_seqno;
   2813 
   2814 	return ret;
   2815 }
   2816 
   2817 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
   2818 {
   2819 	u32 old_write_domain, old_read_domains;
   2820 
   2821 	/* Force a pagefault for domain tracking on next user access */
   2822 	i915_gem_release_mmap(obj);
   2823 
   2824 	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
   2825 		return;
   2826 
   2827 	/* Wait for any direct GTT access to complete */
   2828 	mb();
   2829 
   2830 	old_read_domains = obj->base.read_domains;
   2831 	old_write_domain = obj->base.write_domain;
   2832 
   2833 	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
   2834 	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
   2835 
   2836 	trace_i915_gem_object_change_domain(obj,
   2837 					    old_read_domains,
   2838 					    old_write_domain);
   2839 }
   2840 
   2841 int i915_vma_unbind(struct i915_vma *vma)
   2842 {
   2843 	struct drm_i915_gem_object *obj = vma->obj;
   2844 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
   2845 	int ret;
   2846 
   2847 	if (list_empty(&vma->vma_link))
   2848 		return 0;
   2849 
   2850 	if (!drm_mm_node_allocated(&vma->node)) {
   2851 		i915_gem_vma_destroy(vma);
   2852 		return 0;
   2853 	}
   2854 
   2855 	if (vma->pin_count)
   2856 		return -EBUSY;
   2857 
   2858 	BUG_ON(obj->pages == NULL);
   2859 
   2860 	ret = i915_gem_object_finish_gpu(obj);
   2861 	if (ret)
   2862 		return ret;
   2863 	/* Continue on if we fail due to EIO, the GPU is hung so we
   2864 	 * should be safe and we need to cleanup or else we might
   2865 	 * cause memory corruption through use-after-free.
   2866 	 */
   2867 
   2868 	i915_gem_object_finish_gtt(obj);
   2869 
   2870 	/* release the fence reg _after_ flushing */
   2871 	ret = i915_gem_object_put_fence(obj);
   2872 	if (ret)
   2873 		return ret;
   2874 
   2875 	trace_i915_vma_unbind(vma);
   2876 
   2877 	vma->unbind_vma(vma);
   2878 
   2879 	i915_gem_gtt_finish_object(obj);
   2880 
   2881 	list_del_init(&vma->mm_list);
   2882 	/* Avoid an unnecessary call to unbind on rebind. */
   2883 	if (i915_is_ggtt(vma->vm))
   2884 		obj->map_and_fenceable = true;
   2885 
   2886 	drm_mm_remove_node(&vma->node);
   2887 	i915_gem_vma_destroy(vma);
   2888 
   2889 	/* Since the unbound list is global, only move to that list if
   2890 	 * no more VMAs exist. */
   2891 	if (list_empty(&obj->vma_list))
   2892 		list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
   2893 
   2894 	/* And finally now the object is completely decoupled from this vma,
   2895 	 * we can drop its hold on the backing storage and allow it to be
   2896 	 * reaped by the shrinker.
   2897 	 */
   2898 	i915_gem_object_unpin_pages(obj);
   2899 
   2900 	return 0;
   2901 }
   2902 
   2903 int i915_gpu_idle(struct drm_device *dev)
   2904 {
   2905 	struct drm_i915_private *dev_priv = dev->dev_private;
   2906 	struct intel_ring_buffer *ring;
   2907 	int ret, i;
   2908 
   2909 	/* Flush everything onto the inactive list. */
   2910 	for_each_ring(ring, dev_priv, i) {
   2911 		ret = i915_switch_context(ring, ring->default_context);
   2912 		if (ret)
   2913 			return ret;
   2914 
   2915 		ret = intel_ring_idle(ring);
   2916 		if (ret)
   2917 			return ret;
   2918 	}
   2919 
   2920 	return 0;
   2921 }
   2922 
   2923 static void i965_write_fence_reg(struct drm_device *dev, int reg,
   2924 				 struct drm_i915_gem_object *obj)
   2925 {
   2926 	struct drm_i915_private *dev_priv = dev->dev_private;
   2927 	int fence_reg;
   2928 	int fence_pitch_shift;
   2929 
   2930 	if (INTEL_INFO(dev)->gen >= 6) {
   2931 		fence_reg = FENCE_REG_SANDYBRIDGE_0;
   2932 		fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
   2933 	} else {
   2934 		fence_reg = FENCE_REG_965_0;
   2935 		fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
   2936 	}
   2937 
   2938 	fence_reg += reg * 8;
   2939 
   2940 	/* To w/a incoherency with non-atomic 64-bit register updates,
   2941 	 * we split the 64-bit update into two 32-bit writes. In order
   2942 	 * for a partial fence not to be evaluated between writes, we
   2943 	 * precede the update with write to turn off the fence register,
   2944 	 * and only enable the fence as the last step.
   2945 	 *
   2946 	 * For extra levels of paranoia, we make sure each step lands
   2947 	 * before applying the next step.
   2948 	 */
   2949 	I915_WRITE(fence_reg, 0);
   2950 	POSTING_READ(fence_reg);
   2951 
   2952 	if (obj) {
   2953 		u32 size = i915_gem_obj_ggtt_size(obj);
   2954 		uint64_t val;
   2955 
   2956 		val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
   2957 				 0xfffff000) << 32;
   2958 		val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
   2959 		val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
   2960 		if (obj->tiling_mode == I915_TILING_Y)
   2961 			val |= 1 << I965_FENCE_TILING_Y_SHIFT;
   2962 		val |= I965_FENCE_REG_VALID;
   2963 
   2964 		I915_WRITE(fence_reg + 4, val >> 32);
   2965 		POSTING_READ(fence_reg + 4);
   2966 
   2967 		I915_WRITE(fence_reg + 0, val);
   2968 		POSTING_READ(fence_reg);
   2969 	} else {
   2970 		I915_WRITE(fence_reg + 4, 0);
   2971 		POSTING_READ(fence_reg + 4);
   2972 	}
   2973 }
   2974 
   2975 static void i915_write_fence_reg(struct drm_device *dev, int reg,
   2976 				 struct drm_i915_gem_object *obj)
   2977 {
   2978 	struct drm_i915_private *dev_priv = dev->dev_private;
   2979 	u32 val;
   2980 
   2981 	if (obj) {
   2982 		u32 size = i915_gem_obj_ggtt_size(obj);
   2983 		int pitch_val;
   2984 		int tile_width;
   2985 
   2986 		WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
   2987 		     (size & -size) != size ||
   2988 		     (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
   2989 		     "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
   2990 		     i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
   2991 
   2992 		if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
   2993 			tile_width = 128;
   2994 		else
   2995 			tile_width = 512;
   2996 
   2997 		/* Note: pitch better be a power of two tile widths */
   2998 		pitch_val = obj->stride / tile_width;
   2999 		pitch_val = ffs(pitch_val) - 1;
   3000 
   3001 		val = i915_gem_obj_ggtt_offset(obj);
   3002 		if (obj->tiling_mode == I915_TILING_Y)
   3003 			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
   3004 		val |= I915_FENCE_SIZE_BITS(size);
   3005 		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
   3006 		val |= I830_FENCE_REG_VALID;
   3007 	} else
   3008 		val = 0;
   3009 
   3010 	if (reg < 8)
   3011 		reg = FENCE_REG_830_0 + reg * 4;
   3012 	else
   3013 		reg = FENCE_REG_945_8 + (reg - 8) * 4;
   3014 
   3015 	I915_WRITE(reg, val);
   3016 	POSTING_READ(reg);
   3017 }
   3018 
   3019 static void i830_write_fence_reg(struct drm_device *dev, int reg,
   3020 				struct drm_i915_gem_object *obj)
   3021 {
   3022 	struct drm_i915_private *dev_priv = dev->dev_private;
   3023 	uint32_t val;
   3024 
   3025 	if (obj) {
   3026 		u32 size = i915_gem_obj_ggtt_size(obj);
   3027 		uint32_t pitch_val;
   3028 
   3029 		WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) ||
   3030 		     (size & -size) != size ||
   3031 		     (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
   3032 		     "object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
   3033 		     i915_gem_obj_ggtt_offset(obj), size);
   3034 
   3035 		pitch_val = obj->stride / 128;
   3036 		pitch_val = ffs(pitch_val) - 1;
   3037 
   3038 		val = i915_gem_obj_ggtt_offset(obj);
   3039 		if (obj->tiling_mode == I915_TILING_Y)
   3040 			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
   3041 		val |= I830_FENCE_SIZE_BITS(size);
   3042 		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
   3043 		val |= I830_FENCE_REG_VALID;
   3044 	} else
   3045 		val = 0;
   3046 
   3047 	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
   3048 	POSTING_READ(FENCE_REG_830_0 + reg * 4);
   3049 }
   3050 
   3051 inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
   3052 {
   3053 	return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
   3054 }
   3055 
   3056 static void i915_gem_write_fence(struct drm_device *dev, int reg,
   3057 				 struct drm_i915_gem_object *obj)
   3058 {
   3059 	struct drm_i915_private *dev_priv = dev->dev_private;
   3060 
   3061 	/* Ensure that all CPU reads are completed before installing a fence
   3062 	 * and all writes before removing the fence.
   3063 	 */
   3064 	if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
   3065 		mb();
   3066 
   3067 	WARN(obj && (!obj->stride || !obj->tiling_mode),
   3068 	     "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
   3069 	     obj->stride, obj->tiling_mode);
   3070 
   3071 	switch (INTEL_INFO(dev)->gen) {
   3072 	case 8:
   3073 	case 7:
   3074 	case 6:
   3075 	case 5:
   3076 	case 4: i965_write_fence_reg(dev, reg, obj); break;
   3077 	case 3: i915_write_fence_reg(dev, reg, obj); break;
   3078 	case 2: i830_write_fence_reg(dev, reg, obj); break;
   3079 	default: BUG();
   3080 	}
   3081 
   3082 	/* And similarly be paranoid that no direct access to this region
   3083 	 * is reordered to before the fence is installed.
   3084 	 */
   3085 	if (i915_gem_object_needs_mb(obj))
   3086 		mb();
   3087 }
   3088 
   3089 static inline int fence_number(struct drm_i915_private *dev_priv,
   3090 			       struct drm_i915_fence_reg *fence)
   3091 {
   3092 	return fence - dev_priv->fence_regs;
   3093 }
   3094 
   3095 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
   3096 					 struct drm_i915_fence_reg *fence,
   3097 					 bool enable)
   3098 {
   3099 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
   3100 	int reg = fence_number(dev_priv, fence);
   3101 
   3102 	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
   3103 
   3104 	if (enable) {
   3105 		obj->fence_reg = reg;
   3106 		fence->obj = obj;
   3107 		list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
   3108 	} else {
   3109 		obj->fence_reg = I915_FENCE_REG_NONE;
   3110 		fence->obj = NULL;
   3111 		list_del_init(&fence->lru_list);
   3112 	}
   3113 	obj->fence_dirty = false;
   3114 }
   3115 
   3116 static int
   3117 i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
   3118 {
   3119 	if (obj->last_fenced_seqno) {
   3120 		int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
   3121 		if (ret)
   3122 			return ret;
   3123 
   3124 		obj->last_fenced_seqno = 0;
   3125 	}
   3126 
   3127 	obj->fenced_gpu_access = false;
   3128 	return 0;
   3129 }
   3130 
   3131 int
   3132 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
   3133 {
   3134 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
   3135 	struct drm_i915_fence_reg *fence;
   3136 	int ret;
   3137 
   3138 	ret = i915_gem_object_wait_fence(obj);
   3139 	if (ret)
   3140 		return ret;
   3141 
   3142 	if (obj->fence_reg == I915_FENCE_REG_NONE)
   3143 		return 0;
   3144 
   3145 	fence = &dev_priv->fence_regs[obj->fence_reg];
   3146 
   3147 	i915_gem_object_fence_lost(obj);
   3148 	i915_gem_object_update_fence(obj, fence, false);
   3149 
   3150 	return 0;
   3151 }
   3152 
   3153 static struct drm_i915_fence_reg *
   3154 i915_find_fence_reg(struct drm_device *dev)
   3155 {
   3156 	struct drm_i915_private *dev_priv = dev->dev_private;
   3157 	struct drm_i915_fence_reg *reg, *avail;
   3158 	int i;
   3159 
   3160 	/* First try to find a free reg */
   3161 	avail = NULL;
   3162 	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
   3163 		reg = &dev_priv->fence_regs[i];
   3164 		if (!reg->obj)
   3165 			return reg;
   3166 
   3167 		if (!reg->pin_count)
   3168 			avail = reg;
   3169 	}
   3170 
   3171 	if (avail == NULL)
   3172 		goto deadlock;
   3173 
   3174 	/* None available, try to steal one or wait for a user to finish */
   3175 	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
   3176 		if (reg->pin_count)
   3177 			continue;
   3178 
   3179 		return reg;
   3180 	}
   3181 
   3182 deadlock:
   3183 	/* Wait for completion of pending flips which consume fences */
   3184 	if (intel_has_pending_fb_unpin(dev))
   3185 		return ERR_PTR(-EAGAIN);
   3186 
   3187 	return ERR_PTR(-EDEADLK);
   3188 }
   3189 
   3190 /**
   3191  * i915_gem_object_get_fence - set up fencing for an object
   3192  * @obj: object to map through a fence reg
   3193  *
   3194  * When mapping objects through the GTT, userspace wants to be able to write
   3195  * to them without having to worry about swizzling if the object is tiled.
   3196  * This function walks the fence regs looking for a free one for @obj,
   3197  * stealing one if it can't find any.
   3198  *
   3199  * It then sets up the reg based on the object's properties: address, pitch
   3200  * and tiling format.
   3201  *
   3202  * For an untiled surface, this removes any existing fence.
   3203  */
   3204 int
   3205 i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
   3206 {
   3207 	struct drm_device *dev = obj->base.dev;
   3208 	struct drm_i915_private *dev_priv = dev->dev_private;
   3209 	bool enable = obj->tiling_mode != I915_TILING_NONE;
   3210 	struct drm_i915_fence_reg *reg;
   3211 	int ret;
   3212 
   3213 	/* Have we updated the tiling parameters upon the object and so
   3214 	 * will need to serialise the write to the associated fence register?
   3215 	 */
   3216 	if (obj->fence_dirty) {
   3217 		ret = i915_gem_object_wait_fence(obj);
   3218 		if (ret)
   3219 			return ret;
   3220 	}
   3221 
   3222 	/* Just update our place in the LRU if our fence is getting reused. */
   3223 	if (obj->fence_reg != I915_FENCE_REG_NONE) {
   3224 		reg = &dev_priv->fence_regs[obj->fence_reg];
   3225 		if (!obj->fence_dirty) {
   3226 			list_move_tail(&reg->lru_list,
   3227 				       &dev_priv->mm.fence_list);
   3228 			return 0;
   3229 		}
   3230 	} else if (enable) {
   3231 		reg = i915_find_fence_reg(dev);
   3232 		if (IS_ERR(reg))
   3233 			return PTR_ERR(reg);
   3234 
   3235 		if (reg->obj) {
   3236 			struct drm_i915_gem_object *old = reg->obj;
   3237 
   3238 			ret = i915_gem_object_wait_fence(old);
   3239 			if (ret)
   3240 				return ret;
   3241 
   3242 			i915_gem_object_fence_lost(old);
   3243 		}
   3244 	} else
   3245 		return 0;
   3246 
   3247 	i915_gem_object_update_fence(obj, reg, enable);
   3248 
   3249 	return 0;
   3250 }
   3251 
   3252 static bool i915_gem_valid_gtt_space(struct drm_device *dev,
   3253 				     struct drm_mm_node *gtt_space,
   3254 				     unsigned long cache_level)
   3255 {
   3256 	struct drm_mm_node *other;
   3257 
   3258 	/* On non-LLC machines we have to be careful when putting differing
   3259 	 * types of snoopable memory together to avoid the prefetcher
   3260 	 * crossing memory domains and dying.
   3261 	 */
   3262 	if (HAS_LLC(dev))
   3263 		return true;
   3264 
   3265 	if (!drm_mm_node_allocated(gtt_space))
   3266 		return true;
   3267 
   3268 	if (list_empty(&gtt_space->node_list))
   3269 		return true;
   3270 
   3271 	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
   3272 	if (other->allocated && !other->hole_follows && other->color != cache_level)
   3273 		return false;
   3274 
   3275 	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
   3276 	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
   3277 		return false;
   3278 
   3279 	return true;
   3280 }
   3281 
   3282 static void i915_gem_verify_gtt(struct drm_device *dev)
   3283 {
   3284 #if WATCH_GTT
   3285 	struct drm_i915_private *dev_priv = dev->dev_private;
   3286 	struct drm_i915_gem_object *obj;
   3287 	int err = 0;
   3288 
   3289 	list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
   3290 		if (obj->gtt_space == NULL) {
   3291 			printk(KERN_ERR "object found on GTT list with no space reserved\n");
   3292 			err++;
   3293 			continue;
   3294 		}
   3295 
   3296 		if (obj->cache_level != obj->gtt_space->color) {
   3297 			printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
   3298 			       i915_gem_obj_ggtt_offset(obj),
   3299 			       i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
   3300 			       obj->cache_level,
   3301 			       obj->gtt_space->color);
   3302 			err++;
   3303 			continue;
   3304 		}
   3305 
   3306 		if (!i915_gem_valid_gtt_space(dev,
   3307 					      obj->gtt_space,
   3308 					      obj->cache_level)) {
   3309 			printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
   3310 			       i915_gem_obj_ggtt_offset(obj),
   3311 			       i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
   3312 			       obj->cache_level);
   3313 			err++;
   3314 			continue;
   3315 		}
   3316 	}
   3317 
   3318 	WARN_ON(err);
   3319 #endif
   3320 }
   3321 
   3322 /**
   3323  * Finds free space in the GTT aperture and binds the object there.
   3324  */
   3325 static struct i915_vma *
   3326 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
   3327 			   struct i915_address_space *vm,
   3328 			   unsigned alignment,
   3329 			   uint64_t flags)
   3330 {
   3331 	struct drm_device *dev = obj->base.dev;
   3332 	struct drm_i915_private *dev_priv = dev->dev_private;
   3333 	u32 size, fence_size, fence_alignment, unfenced_alignment;
   3334 	unsigned long start =
   3335 		flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
   3336 	unsigned long end =
   3337 		flags & PIN_MAPPABLE ? dev_priv->gtt.mappable_end : vm->total;
   3338 	struct i915_vma *vma;
   3339 	int ret;
   3340 
   3341 	fence_size = i915_gem_get_gtt_size(dev,
   3342 					   obj->base.size,
   3343 					   obj->tiling_mode);
   3344 	fence_alignment = i915_gem_get_gtt_alignment(dev,
   3345 						     obj->base.size,
   3346 						     obj->tiling_mode, true);
   3347 	unfenced_alignment =
   3348 		i915_gem_get_gtt_alignment(dev,
   3349 					   obj->base.size,
   3350 					   obj->tiling_mode, false);
   3351 
   3352 	if (alignment == 0)
   3353 		alignment = flags & PIN_MAPPABLE ? fence_alignment :
   3354 						unfenced_alignment;
   3355 	if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
   3356 		DRM_DEBUG("Invalid object alignment requested %u\n", alignment);
   3357 		return ERR_PTR(-EINVAL);
   3358 	}
   3359 
   3360 	size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
   3361 
   3362 	/* If the object is bigger than the entire aperture, reject it early
   3363 	 * before evicting everything in a vain attempt to find space.
   3364 	 */
   3365 	if (obj->base.size > end) {
   3366 		DRM_DEBUG("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%lu\n",
   3367 			  obj->base.size,
   3368 			  flags & PIN_MAPPABLE ? "mappable" : "total",
   3369 			  end);
   3370 		return ERR_PTR(-E2BIG);
   3371 	}
   3372 
   3373 	ret = i915_gem_object_get_pages(obj);
   3374 	if (ret)
   3375 		return ERR_PTR(ret);
   3376 
   3377 	i915_gem_object_pin_pages(obj);
   3378 
   3379 	vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
   3380 	if (IS_ERR(vma))
   3381 		goto err_unpin;
   3382 
   3383 search_free:
   3384 	ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
   3385 						  size, alignment,
   3386 						  obj->cache_level,
   3387 						  start, end,
   3388 						  DRM_MM_SEARCH_DEFAULT,
   3389 						  DRM_MM_CREATE_DEFAULT);
   3390 	if (ret) {
   3391 		ret = i915_gem_evict_something(dev, vm, size, alignment,
   3392 					       obj->cache_level,
   3393 					       start, end,
   3394 					       flags);
   3395 		if (ret == 0)
   3396 			goto search_free;
   3397 
   3398 		goto err_free_vma;
   3399 	}
   3400 	if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
   3401 					      obj->cache_level))) {
   3402 		ret = -EINVAL;
   3403 		goto err_remove_node;
   3404 	}
   3405 
   3406 	ret = i915_gem_gtt_prepare_object(obj);
   3407 	if (ret)
   3408 		goto err_remove_node;
   3409 
   3410 	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
   3411 	list_add_tail(&vma->mm_list, &vm->inactive_list);
   3412 
   3413 	if (i915_is_ggtt(vm)) {
   3414 		bool mappable, fenceable;
   3415 
   3416 		fenceable = (vma->node.size == fence_size &&
   3417 			     (vma->node.start & (fence_alignment - 1)) == 0);
   3418 
   3419 		mappable = (vma->node.start + obj->base.size <=
   3420 			    dev_priv->gtt.mappable_end);
   3421 
   3422 		obj->map_and_fenceable = mappable && fenceable;
   3423 	}
   3424 
   3425 	WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
   3426 
   3427 	trace_i915_vma_bind(vma, flags);
   3428 	vma->bind_vma(vma, obj->cache_level,
   3429 		      flags & (PIN_MAPPABLE | PIN_GLOBAL) ? GLOBAL_BIND : 0);
   3430 
   3431 	i915_gem_verify_gtt(dev);
   3432 	return vma;
   3433 
   3434 err_remove_node:
   3435 	drm_mm_remove_node(&vma->node);
   3436 err_free_vma:
   3437 	i915_gem_vma_destroy(vma);
   3438 	vma = ERR_PTR(ret);
   3439 err_unpin:
   3440 	i915_gem_object_unpin_pages(obj);
   3441 	return vma;
   3442 }
   3443 
   3444 bool
   3445 i915_gem_clflush_object(struct drm_i915_gem_object *obj,
   3446 			bool force)
   3447 {
   3448 	/* If we don't have a page list set up, then we're not pinned
   3449 	 * to GPU, and we can ignore the cache flush because it'll happen
   3450 	 * again at bind time.
   3451 	 */
   3452 	if (obj->pages == NULL)
   3453 		return false;
   3454 
   3455 	/*
   3456 	 * Stolen memory is always coherent with the GPU as it is explicitly
   3457 	 * marked as wc by the system, or the system is cache-coherent.
   3458 	 */
   3459 	if (obj->stolen)
   3460 		return false;
   3461 
   3462 	/* If the GPU is snooping the contents of the CPU cache,
   3463 	 * we do not need to manually clear the CPU cache lines.  However,
   3464 	 * the caches are only snooped when the render cache is
   3465 	 * flushed/invalidated.  As we always have to emit invalidations
   3466 	 * and flushes when moving into and out of the RENDER domain, correct
   3467 	 * snooping behaviour occurs naturally as the result of our domain
   3468 	 * tracking.
   3469 	 */
   3470 	if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
   3471 		return false;
   3472 
   3473 	trace_i915_gem_object_clflush(obj);
   3474 	drm_clflush_sg(obj->pages);
   3475 
   3476 	return true;
   3477 }
   3478 
   3479 /** Flushes the GTT write domain for the object if it's dirty. */
   3480 static void
   3481 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
   3482 {
   3483 	uint32_t old_write_domain;
   3484 
   3485 	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
   3486 		return;
   3487 
   3488 	/* No actual flushing is required for the GTT write domain.  Writes
   3489 	 * to it immediately go to main memory as far as we know, so there's
   3490 	 * no chipset flush.  It also doesn't land in render cache.
   3491 	 *
   3492 	 * However, we do have to enforce the order so that all writes through
   3493 	 * the GTT land before any writes to the device, such as updates to
   3494 	 * the GATT itself.
   3495 	 */
   3496 	wmb();
   3497 
   3498 	old_write_domain = obj->base.write_domain;
   3499 	obj->base.write_domain = 0;
   3500 
   3501 	trace_i915_gem_object_change_domain(obj,
   3502 					    obj->base.read_domains,
   3503 					    old_write_domain);
   3504 }
   3505 
   3506 /** Flushes the CPU write domain for the object if it's dirty. */
   3507 static void
   3508 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
   3509 				       bool force)
   3510 {
   3511 	uint32_t old_write_domain;
   3512 
   3513 	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
   3514 		return;
   3515 
   3516 	if (i915_gem_clflush_object(obj, force))
   3517 		i915_gem_chipset_flush(obj->base.dev);
   3518 
   3519 	old_write_domain = obj->base.write_domain;
   3520 	obj->base.write_domain = 0;
   3521 
   3522 	trace_i915_gem_object_change_domain(obj,
   3523 					    obj->base.read_domains,
   3524 					    old_write_domain);
   3525 }
   3526 
   3527 /**
   3528  * Moves a single object to the GTT read, and possibly write domain.
   3529  *
   3530  * This function returns when the move is complete, including waiting on
   3531  * flushes to occur.
   3532  */
   3533 int
   3534 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
   3535 {
   3536 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
   3537 	uint32_t old_write_domain, old_read_domains;
   3538 	int ret;
   3539 
   3540 	/* Not valid to be called on unbound objects. */
   3541 	if (!i915_gem_obj_bound_any(obj))
   3542 		return -EINVAL;
   3543 
   3544 	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
   3545 		return 0;
   3546 
   3547 	ret = i915_gem_object_wait_rendering(obj, !write);
   3548 	if (ret)
   3549 		return ret;
   3550 
   3551 	i915_gem_object_flush_cpu_write_domain(obj, false);
   3552 
   3553 	/* Serialise direct access to this object with the barriers for
   3554 	 * coherent writes from the GPU, by effectively invalidating the
   3555 	 * GTT domain upon first access.
   3556 	 */
   3557 	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
   3558 		mb();
   3559 
   3560 	old_write_domain = obj->base.write_domain;
   3561 	old_read_domains = obj->base.read_domains;
   3562 
   3563 	/* It should now be out of any other write domains, and we can update
   3564 	 * the domain values for our changes.
   3565 	 */
   3566 	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
   3567 	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
   3568 	if (write) {
   3569 		obj->base.read_domains = I915_GEM_DOMAIN_GTT;
   3570 		obj->base.write_domain = I915_GEM_DOMAIN_GTT;
   3571 		obj->dirty = 1;
   3572 	}
   3573 
   3574 	trace_i915_gem_object_change_domain(obj,
   3575 					    old_read_domains,
   3576 					    old_write_domain);
   3577 
   3578 	/* And bump the LRU for this access */
   3579 	if (i915_gem_object_is_inactive(obj)) {
   3580 		struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
   3581 		if (vma)
   3582 			list_move_tail(&vma->mm_list,
   3583 				       &dev_priv->gtt.base.inactive_list);
   3584 
   3585 	}
   3586 
   3587 	return 0;
   3588 }
   3589 
   3590 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
   3591 				    enum i915_cache_level cache_level)
   3592 {
   3593 	struct drm_device *dev = obj->base.dev;
   3594 	struct i915_vma *vma, *next;
   3595 	int ret;
   3596 
   3597 	if (obj->cache_level == cache_level)
   3598 		return 0;
   3599 
   3600 	if (i915_gem_obj_is_pinned(obj)) {
   3601 		DRM_DEBUG("can not change the cache level of pinned objects\n");
   3602 		return -EBUSY;
   3603 	}
   3604 
   3605 	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
   3606 		if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
   3607 			ret = i915_vma_unbind(vma);
   3608 			if (ret)
   3609 				return ret;
   3610 		}
   3611 	}
   3612 
   3613 	if (i915_gem_obj_bound_any(obj)) {
   3614 		ret = i915_gem_object_finish_gpu(obj);
   3615 		if (ret)
   3616 			return ret;
   3617 
   3618 		i915_gem_object_finish_gtt(obj);
   3619 
   3620 		/* Before SandyBridge, you could not use tiling or fence
   3621 		 * registers with snooped memory, so relinquish any fences
   3622 		 * currently pointing to our region in the aperture.
   3623 		 */
   3624 		if (INTEL_INFO(dev)->gen < 6) {
   3625 			ret = i915_gem_object_put_fence(obj);
   3626 			if (ret)
   3627 				return ret;
   3628 		}
   3629 
   3630 		list_for_each_entry(vma, &obj->vma_list, vma_link)
   3631 			if (drm_mm_node_allocated(&vma->node))
   3632 				vma->bind_vma(vma, cache_level,
   3633 					      obj->has_global_gtt_mapping ? GLOBAL_BIND : 0);
   3634 	}
   3635 
   3636 	list_for_each_entry(vma, &obj->vma_list, vma_link)
   3637 		vma->node.color = cache_level;
   3638 	obj->cache_level = cache_level;
   3639 
   3640 	if (cpu_write_needs_clflush(obj)) {
   3641 		u32 old_read_domains, old_write_domain;
   3642 
   3643 		/* If we're coming from LLC cached, then we haven't
   3644 		 * actually been tracking whether the data is in the
   3645 		 * CPU cache or not, since we only allow one bit set
   3646 		 * in obj->write_domain and have been skipping the clflushes.
   3647 		 * Just set it to the CPU cache for now.
   3648 		 */
   3649 		WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
   3650 
   3651 		old_read_domains = obj->base.read_domains;
   3652 		old_write_domain = obj->base.write_domain;
   3653 
   3654 		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
   3655 		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
   3656 
   3657 		trace_i915_gem_object_change_domain(obj,
   3658 						    old_read_domains,
   3659 						    old_write_domain);
   3660 	}
   3661 
   3662 	i915_gem_verify_gtt(dev);
   3663 	return 0;
   3664 }
   3665 
   3666 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
   3667 			       struct drm_file *file)
   3668 {
   3669 	struct drm_i915_gem_caching *args = data;
   3670 	struct drm_i915_gem_object *obj;
   3671 	int ret;
   3672 
   3673 	ret = i915_mutex_lock_interruptible(dev);
   3674 	if (ret)
   3675 		return ret;
   3676 
   3677 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
   3678 	if (&obj->base == NULL) {
   3679 		ret = -ENOENT;
   3680 		goto unlock;
   3681 	}
   3682 
   3683 	switch (obj->cache_level) {
   3684 	case I915_CACHE_LLC:
   3685 	case I915_CACHE_L3_LLC:
   3686 		args->caching = I915_CACHING_CACHED;
   3687 		break;
   3688 
   3689 	case I915_CACHE_WT:
   3690 		args->caching = I915_CACHING_DISPLAY;
   3691 		break;
   3692 
   3693 	default:
   3694 		args->caching = I915_CACHING_NONE;
   3695 		break;
   3696 	}
   3697 
   3698 	drm_gem_object_unreference(&obj->base);
   3699 unlock:
   3700 	mutex_unlock(&dev->struct_mutex);
   3701 	return ret;
   3702 }
   3703 
   3704 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
   3705 			       struct drm_file *file)
   3706 {
   3707 	struct drm_i915_gem_caching *args = data;
   3708 	struct drm_i915_gem_object *obj;
   3709 	enum i915_cache_level level;
   3710 	int ret;
   3711 
   3712 	switch (args->caching) {
   3713 	case I915_CACHING_NONE:
   3714 		level = I915_CACHE_NONE;
   3715 		break;
   3716 	case I915_CACHING_CACHED:
   3717 		level = I915_CACHE_LLC;
   3718 		break;
   3719 	case I915_CACHING_DISPLAY:
   3720 		level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
   3721 		break;
   3722 	default:
   3723 		return -EINVAL;
   3724 	}
   3725 
   3726 	ret = i915_mutex_lock_interruptible(dev);
   3727 	if (ret)
   3728 		return ret;
   3729 
   3730 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
   3731 	if (&obj->base == NULL) {
   3732 		ret = -ENOENT;
   3733 		goto unlock;
   3734 	}
   3735 
   3736 	ret = i915_gem_object_set_cache_level(obj, level);
   3737 
   3738 	drm_gem_object_unreference(&obj->base);
   3739 unlock:
   3740 	mutex_unlock(&dev->struct_mutex);
   3741 	return ret;
   3742 }
   3743 
   3744 static bool is_pin_display(struct drm_i915_gem_object *obj)
   3745 {
   3746 	/* There are 3 sources that pin objects:
   3747 	 *   1. The display engine (scanouts, sprites, cursors);
   3748 	 *   2. Reservations for execbuffer;
   3749 	 *   3. The user.
   3750 	 *
   3751 	 * We can ignore reservations as we hold the struct_mutex and
   3752 	 * are only called outside of the reservation path.  The user
   3753 	 * can only increment pin_count once, and so if after
   3754 	 * subtracting the potential reference by the user, any pin_count
   3755 	 * remains, it must be due to another use by the display engine.
   3756 	 */
   3757 	return i915_gem_obj_to_ggtt(obj)->pin_count - !!obj->user_pin_count;
   3758 }
   3759 
   3760 /*
   3761  * Prepare buffer for display plane (scanout, cursors, etc).
   3762  * Can be called from an uninterruptible phase (modesetting) and allows
   3763  * any flushes to be pipelined (for pageflips).
   3764  */
   3765 int
   3766 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
   3767 				     u32 alignment,
   3768 				     struct intel_ring_buffer *pipelined)
   3769 {
   3770 	u32 old_read_domains, old_write_domain;
   3771 	int ret;
   3772 
   3773 	if (pipelined != obj->ring) {
   3774 		ret = i915_gem_object_sync(obj, pipelined);
   3775 		if (ret)
   3776 			return ret;
   3777 	}
   3778 
   3779 	/* Mark the pin_display early so that we account for the
   3780 	 * display coherency whilst setting up the cache domains.
   3781 	 */
   3782 	obj->pin_display = true;
   3783 
   3784 	/* The display engine is not coherent with the LLC cache on gen6.  As
   3785 	 * a result, we make sure that the pinning that is about to occur is
   3786 	 * done with uncached PTEs. This is lowest common denominator for all
   3787 	 * chipsets.
   3788 	 *
   3789 	 * However for gen6+, we could do better by using the GFDT bit instead
   3790 	 * of uncaching, which would allow us to flush all the LLC-cached data
   3791 	 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
   3792 	 */
   3793 	ret = i915_gem_object_set_cache_level(obj,
   3794 					      HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
   3795 	if (ret)
   3796 		goto err_unpin_display;
   3797 
   3798 	/* As the user may map the buffer once pinned in the display plane
   3799 	 * (e.g. libkms for the bootup splash), we have to ensure that we
   3800 	 * always use map_and_fenceable for all scanout buffers.
   3801 	 */
   3802 	ret = i915_gem_obj_ggtt_pin(obj, alignment, PIN_MAPPABLE);
   3803 	if (ret)
   3804 		goto err_unpin_display;
   3805 
   3806 	i915_gem_object_flush_cpu_write_domain(obj, true);
   3807 
   3808 	old_write_domain = obj->base.write_domain;
   3809 	old_read_domains = obj->base.read_domains;
   3810 
   3811 	/* It should now be out of any other write domains, and we can update
   3812 	 * the domain values for our changes.
   3813 	 */
   3814 	obj->base.write_domain = 0;
   3815 	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
   3816 
   3817 	trace_i915_gem_object_change_domain(obj,
   3818 					    old_read_domains,
   3819 					    old_write_domain);
   3820 
   3821 	return 0;
   3822 
   3823 err_unpin_display:
   3824 	obj->pin_display = is_pin_display(obj);
   3825 	return ret;
   3826 }
   3827 
   3828 void
   3829 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
   3830 {
   3831 	i915_gem_object_ggtt_unpin(obj);
   3832 	obj->pin_display = is_pin_display(obj);
   3833 }
   3834 
   3835 int
   3836 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
   3837 {
   3838 	int ret;
   3839 
   3840 	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
   3841 		return 0;
   3842 
   3843 	ret = i915_gem_object_wait_rendering(obj, false);
   3844 	if (ret)
   3845 		return ret;
   3846 
   3847 	/* Ensure that we invalidate the GPU's caches and TLBs. */
   3848 	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
   3849 	return 0;
   3850 }
   3851 
   3852 /**
   3853  * Moves a single object to the CPU read, and possibly write domain.
   3854  *
   3855  * This function returns when the move is complete, including waiting on
   3856  * flushes to occur.
   3857  */
   3858 int
   3859 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
   3860 {
   3861 	uint32_t old_write_domain, old_read_domains;
   3862 	int ret;
   3863 
   3864 	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
   3865 		return 0;
   3866 
   3867 	ret = i915_gem_object_wait_rendering(obj, !write);
   3868 	if (ret)
   3869 		return ret;
   3870 
   3871 	i915_gem_object_flush_gtt_write_domain(obj);
   3872 
   3873 	old_write_domain = obj->base.write_domain;
   3874 	old_read_domains = obj->base.read_domains;
   3875 
   3876 	/* Flush the CPU cache if it's still invalid. */
   3877 	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
   3878 		i915_gem_clflush_object(obj, false);
   3879 
   3880 		obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
   3881 	}
   3882 
   3883 	/* It should now be out of any other write domains, and we can update
   3884 	 * the domain values for our changes.
   3885 	 */
   3886 	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
   3887 
   3888 	/* If we're writing through the CPU, then the GPU read domains will
   3889 	 * need to be invalidated at next use.
   3890 	 */
   3891 	if (write) {
   3892 		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
   3893 		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
   3894 	}
   3895 
   3896 	trace_i915_gem_object_change_domain(obj,
   3897 					    old_read_domains,
   3898 					    old_write_domain);
   3899 
   3900 	return 0;
   3901 }
   3902 
   3903 /* Throttle our rendering by waiting until the ring has completed our requests
   3904  * emitted over 20 msec ago.
   3905  *
   3906  * Note that if we were to use the current jiffies each time around the loop,
   3907  * we wouldn't escape the function with any frames outstanding if the time to
   3908  * render a frame was over 20ms.
   3909  *
   3910  * This should get us reasonable parallelism between CPU and GPU but also
   3911  * relatively low latency when blocking on a particular request to finish.
   3912  */
   3913 static int
   3914 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
   3915 {
   3916 	struct drm_i915_private *dev_priv = dev->dev_private;
   3917 	struct drm_i915_file_private *file_priv = file->driver_priv;
   3918 	unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
   3919 	struct drm_i915_gem_request *request;
   3920 	struct intel_ring_buffer *ring = NULL;
   3921 	unsigned reset_counter;
   3922 	u32 seqno = 0;
   3923 	int ret;
   3924 
   3925 	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
   3926 	if (ret)
   3927 		return ret;
   3928 
   3929 	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
   3930 	if (ret)
   3931 		return ret;
   3932 
   3933 	spin_lock(&file_priv->mm.lock);
   3934 	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
   3935 		if (time_after_eq(request->emitted_jiffies, recent_enough))
   3936 			break;
   3937 
   3938 		ring = request->ring;
   3939 		seqno = request->seqno;
   3940 	}
   3941 	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
   3942 	spin_unlock(&file_priv->mm.lock);
   3943 
   3944 	if (seqno == 0)
   3945 		return 0;
   3946 
   3947 	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, NULL);
   3948 	if (ret == 0)
   3949 		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
   3950 
   3951 	return ret;
   3952 }
   3953 
   3954 static bool
   3955 i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
   3956 {
   3957 	struct drm_i915_gem_object *obj = vma->obj;
   3958 
   3959 	if (alignment &&
   3960 	    vma->node.start & (alignment - 1))
   3961 		return true;
   3962 
   3963 	if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
   3964 		return true;
   3965 
   3966 	if (flags & PIN_OFFSET_BIAS &&
   3967 	    vma->node.start < (flags & PIN_OFFSET_MASK))
   3968 		return true;
   3969 
   3970 	return false;
   3971 }
   3972 
   3973 int
   3974 i915_gem_object_pin(struct drm_i915_gem_object *obj,
   3975 		    struct i915_address_space *vm,
   3976 		    uint32_t alignment,
   3977 		    uint64_t flags)
   3978 {
   3979 	struct i915_vma *vma;
   3980 	int ret;
   3981 
   3982 	if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
   3983 		return -EINVAL;
   3984 
   3985 	vma = i915_gem_obj_to_vma(obj, vm);
   3986 	if (vma) {
   3987 		if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
   3988 			return -EBUSY;
   3989 
   3990 		if (i915_vma_misplaced(vma, alignment, flags)) {
   3991 			WARN(vma->pin_count,
   3992 			     "bo is already pinned with incorrect alignment:"
   3993 			     " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
   3994 			     " obj->map_and_fenceable=%d\n",
   3995 			     i915_gem_obj_offset(obj, vm), alignment,
   3996 			     !!(flags & PIN_MAPPABLE),
   3997 			     obj->map_and_fenceable);
   3998 			ret = i915_vma_unbind(vma);
   3999 			if (ret)
   4000 				return ret;
   4001 
   4002 			vma = NULL;
   4003 		}
   4004 	}
   4005 
   4006 	if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
   4007 		vma = i915_gem_object_bind_to_vm(obj, vm, alignment, flags);
   4008 		if (IS_ERR(vma))
   4009 			return PTR_ERR(vma);
   4010 	}
   4011 
   4012 	if (flags & PIN_GLOBAL && !obj->has_global_gtt_mapping)
   4013 		vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
   4014 
   4015 	vma->pin_count++;
   4016 	if (flags & PIN_MAPPABLE)
   4017 		obj->pin_mappable |= true;
   4018 
   4019 	return 0;
   4020 }
   4021 
   4022 void
   4023 i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj)
   4024 {
   4025 	struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
   4026 
   4027 	BUG_ON(!vma);
   4028 	BUG_ON(vma->pin_count == 0);
   4029 	BUG_ON(!i915_gem_obj_ggtt_bound(obj));
   4030 
   4031 	if (--vma->pin_count == 0)
   4032 		obj->pin_mappable = false;
   4033 }
   4034 
   4035 int
   4036 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
   4037 		   struct drm_file *file)
   4038 {
   4039 	struct drm_i915_gem_pin *args = data;
   4040 	struct drm_i915_gem_object *obj;
   4041 	int ret;
   4042 
   4043 	if (INTEL_INFO(dev)->gen >= 6)
   4044 		return -ENODEV;
   4045 
   4046 	ret = i915_mutex_lock_interruptible(dev);
   4047 	if (ret)
   4048 		return ret;
   4049 
   4050 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
   4051 	if (&obj->base == NULL) {
   4052 		ret = -ENOENT;
   4053 		goto unlock;
   4054 	}
   4055 
   4056 	if (obj->madv != I915_MADV_WILLNEED) {
   4057 		DRM_DEBUG("Attempting to pin a purgeable buffer\n");
   4058 		ret = -EFAULT;
   4059 		goto out;
   4060 	}
   4061 
   4062 	if (obj->pin_filp != NULL && obj->pin_filp != file) {
   4063 		DRM_DEBUG("Already pinned in i915_gem_pin_ioctl(): %d\n",
   4064 			  args->handle);
   4065 		ret = -EINVAL;
   4066 		goto out;
   4067 	}
   4068 
   4069 	if (obj->user_pin_count == ULONG_MAX) {
   4070 		ret = -EBUSY;
   4071 		goto out;
   4072 	}
   4073 
   4074 	if (obj->user_pin_count == 0) {
   4075 		ret = i915_gem_obj_ggtt_pin(obj, args->alignment, PIN_MAPPABLE);
   4076 		if (ret)
   4077 			goto out;
   4078 	}
   4079 
   4080 	obj->user_pin_count++;
   4081 	obj->pin_filp = file;
   4082 
   4083 	args->offset = i915_gem_obj_ggtt_offset(obj);
   4084 out:
   4085 	drm_gem_object_unreference(&obj->base);
   4086 unlock:
   4087 	mutex_unlock(&dev->struct_mutex);
   4088 	return ret;
   4089 }
   4090 
   4091 int
   4092 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
   4093 		     struct drm_file *file)
   4094 {
   4095 	struct drm_i915_gem_pin *args = data;
   4096 	struct drm_i915_gem_object *obj;
   4097 	int ret;
   4098 
   4099 	ret = i915_mutex_lock_interruptible(dev);
   4100 	if (ret)
   4101 		return ret;
   4102 
   4103 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
   4104 	if (&obj->base == NULL) {
   4105 		ret = -ENOENT;
   4106 		goto unlock;
   4107 	}
   4108 
   4109 	if (obj->pin_filp != file) {
   4110 		DRM_DEBUG("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
   4111 			  args->handle);
   4112 		ret = -EINVAL;
   4113 		goto out;
   4114 	}
   4115 	obj->user_pin_count--;
   4116 	if (obj->user_pin_count == 0) {
   4117 		obj->pin_filp = NULL;
   4118 		i915_gem_object_ggtt_unpin(obj);
   4119 	}
   4120 
   4121 out:
   4122 	drm_gem_object_unreference(&obj->base);
   4123 unlock:
   4124 	mutex_unlock(&dev->struct_mutex);
   4125 	return ret;
   4126 }
   4127 
   4128 int
   4129 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
   4130 		    struct drm_file *file)
   4131 {
   4132 	struct drm_i915_gem_busy *args = data;
   4133 	struct drm_i915_gem_object *obj;
   4134 	int ret;
   4135 
   4136 	ret = i915_mutex_lock_interruptible(dev);
   4137 	if (ret)
   4138 		return ret;
   4139 
   4140 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
   4141 	if (&obj->base == NULL) {
   4142 		ret = -ENOENT;
   4143 		goto unlock;
   4144 	}
   4145 
   4146 	/* Count all active objects as busy, even if they are currently not used
   4147 	 * by the gpu. Users of this interface expect objects to eventually
   4148 	 * become non-busy without any further actions, therefore emit any
   4149 	 * necessary flushes here.
   4150 	 */
   4151 	ret = i915_gem_object_flush_active(obj);
   4152 
   4153 	args->busy = obj->active;
   4154 	if (obj->ring) {
   4155 		BUILD_BUG_ON(I915_NUM_RINGS > 16);
   4156 		args->busy |= intel_ring_flag(obj->ring) << 16;
   4157 	}
   4158 
   4159 	drm_gem_object_unreference(&obj->base);
   4160 unlock:
   4161 	mutex_unlock(&dev->struct_mutex);
   4162 	return ret;
   4163 }
   4164 
   4165 int
   4166 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
   4167 			struct drm_file *file_priv)
   4168 {
   4169 	return i915_gem_ring_throttle(dev, file_priv);
   4170 }
   4171 
   4172 int
   4173 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
   4174 		       struct drm_file *file_priv)
   4175 {
   4176 	struct drm_i915_gem_madvise *args = data;
   4177 	struct drm_i915_gem_object *obj;
   4178 	int ret;
   4179 
   4180 	switch (args->madv) {
   4181 	case I915_MADV_DONTNEED:
   4182 	case I915_MADV_WILLNEED:
   4183 	    break;
   4184 	default:
   4185 	    return -EINVAL;
   4186 	}
   4187 
   4188 	ret = i915_mutex_lock_interruptible(dev);
   4189 	if (ret)
   4190 		return ret;
   4191 
   4192 	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
   4193 	if (&obj->base == NULL) {
   4194 		ret = -ENOENT;
   4195 		goto unlock;
   4196 	}
   4197 
   4198 	if (i915_gem_obj_is_pinned(obj)) {
   4199 		ret = -EINVAL;
   4200 		goto out;
   4201 	}
   4202 
   4203 	if (obj->madv != __I915_MADV_PURGED)
   4204 		obj->madv = args->madv;
   4205 
   4206 	/* if the object is no longer attached, discard its backing storage */
   4207 	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
   4208 		i915_gem_object_truncate(obj);
   4209 
   4210 	args->retained = obj->madv != __I915_MADV_PURGED;
   4211 
   4212 out:
   4213 	drm_gem_object_unreference(&obj->base);
   4214 unlock:
   4215 	mutex_unlock(&dev->struct_mutex);
   4216 	return ret;
   4217 }
   4218 
   4219 void i915_gem_object_init(struct drm_i915_gem_object *obj,
   4220 			  const struct drm_i915_gem_object_ops *ops)
   4221 {
   4222 	INIT_LIST_HEAD(&obj->global_list);
   4223 	INIT_LIST_HEAD(&obj->ring_list);
   4224 	INIT_LIST_HEAD(&obj->obj_exec_link);
   4225 	INIT_LIST_HEAD(&obj->vma_list);
   4226 
   4227 	obj->ops = ops;
   4228 
   4229 	obj->fence_reg = I915_FENCE_REG_NONE;
   4230 	obj->madv = I915_MADV_WILLNEED;
   4231 	/* Avoid an unnecessary call to unbind on the first bind. */
   4232 	obj->map_and_fenceable = true;
   4233 
   4234 	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
   4235 }
   4236 
   4237 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
   4238 	.get_pages = i915_gem_object_get_pages_gtt,
   4239 	.put_pages = i915_gem_object_put_pages_gtt,
   4240 };
   4241 
   4242 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
   4243 						  size_t size)
   4244 {
   4245 	struct drm_i915_gem_object *obj;
   4246 	struct address_space *mapping;
   4247 	gfp_t mask;
   4248 
   4249 	obj = i915_gem_object_alloc(dev);
   4250 	if (obj == NULL)
   4251 		return NULL;
   4252 
   4253 	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
   4254 		i915_gem_object_free(obj);
   4255 		return NULL;
   4256 	}
   4257 
   4258 	mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
   4259 	if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
   4260 		/* 965gm cannot relocate objects above 4GiB. */
   4261 		mask &= ~__GFP_HIGHMEM;
   4262 		mask |= __GFP_DMA32;
   4263 	}
   4264 
   4265 	mapping = file_inode(obj->base.filp)->i_mapping;
   4266 	mapping_set_gfp_mask(mapping, mask);
   4267 
   4268 	i915_gem_object_init(obj, &i915_gem_object_ops);
   4269 
   4270 	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
   4271 	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
   4272 
   4273 	if (HAS_LLC(dev)) {
   4274 		/* On some devices, we can have the GPU use the LLC (the CPU
   4275 		 * cache) for about a 10% performance improvement
   4276 		 * compared to uncached.  Graphics requests other than
   4277 		 * display scanout are coherent with the CPU in
   4278 		 * accessing this cache.  This means in this mode we
   4279 		 * don't need to clflush on the CPU side, and on the
   4280 		 * GPU side we only need to flush internal caches to
   4281 		 * get data visible to the CPU.
   4282 		 *
   4283 		 * However, we maintain the display planes as UC, and so
   4284 		 * need to rebind when first used as such.
   4285 		 */
   4286 		obj->cache_level = I915_CACHE_LLC;
   4287 	} else
   4288 		obj->cache_level = I915_CACHE_NONE;
   4289 
   4290 	trace_i915_gem_object_create(obj);
   4291 
   4292 	return obj;
   4293 }
   4294 
   4295 void i915_gem_free_object(struct drm_gem_object *gem_obj)
   4296 {
   4297 	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
   4298 	struct drm_device *dev = obj->base.dev;
   4299 	struct drm_i915_private *dev_priv = dev->dev_private;
   4300 	struct i915_vma *vma, *next;
   4301 
   4302 	intel_runtime_pm_get(dev_priv);
   4303 
   4304 	trace_i915_gem_object_destroy(obj);
   4305 
   4306 	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
   4307 		int ret;
   4308 
   4309 		vma->pin_count = 0;
   4310 		ret = i915_vma_unbind(vma);
   4311 		if (WARN_ON(ret == -ERESTARTSYS)) {
   4312 			bool was_interruptible;
   4313 
   4314 			was_interruptible = dev_priv->mm.interruptible;
   4315 			dev_priv->mm.interruptible = false;
   4316 
   4317 			WARN_ON(i915_vma_unbind(vma));
   4318 
   4319 			dev_priv->mm.interruptible = was_interruptible;
   4320 		}
   4321 	}
   4322 
   4323 	i915_gem_object_detach_phys(obj);
   4324 
   4325 	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
   4326 	 * before progressing. */
   4327 	if (obj->stolen)
   4328 		i915_gem_object_unpin_pages(obj);
   4329 
   4330 	if (WARN_ON(obj->pages_pin_count))
   4331 		obj->pages_pin_count = 0;
   4332 	i915_gem_object_put_pages(obj);
   4333 	i915_gem_object_free_mmap_offset(obj);
   4334 	i915_gem_object_release_stolen(obj);
   4335 
   4336 	BUG_ON(obj->pages);
   4337 
   4338 	if (obj->base.import_attach)
   4339 		drm_prime_gem_destroy(&obj->base, NULL);
   4340 
   4341 	drm_gem_object_release(&obj->base);
   4342 	i915_gem_info_remove_obj(dev_priv, obj->base.size);
   4343 
   4344 	kfree(obj->bit_17);
   4345 	i915_gem_object_free(obj);
   4346 
   4347 	intel_runtime_pm_put(dev_priv);
   4348 }
   4349 
   4350 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
   4351 				     struct i915_address_space *vm)
   4352 {
   4353 	struct i915_vma *vma;
   4354 	list_for_each_entry(vma, &obj->vma_list, vma_link)
   4355 		if (vma->vm == vm)
   4356 			return vma;
   4357 
   4358 	return NULL;
   4359 }
   4360 
   4361 void i915_gem_vma_destroy(struct i915_vma *vma)
   4362 {
   4363 	WARN_ON(vma->node.allocated);
   4364 
   4365 	/* Keep the vma as a placeholder in the execbuffer reservation lists */
   4366 	if (!list_empty(&vma->exec_list))
   4367 		return;
   4368 
   4369 	list_del(&vma->vma_link);
   4370 
   4371 	kfree(vma);
   4372 }
   4373 
   4374 int
   4375 i915_gem_suspend(struct drm_device *dev)
   4376 {
   4377 	struct drm_i915_private *dev_priv = dev->dev_private;
   4378 	int ret = 0;
   4379 
   4380 	mutex_lock(&dev->struct_mutex);
   4381 	if (dev_priv->ums.mm_suspended)
   4382 		goto err;
   4383 
   4384 	ret = i915_gpu_idle(dev);
   4385 	if (ret)
   4386 		goto err;
   4387 
   4388 	i915_gem_retire_requests(dev);
   4389 
   4390 	/* Under UMS, be paranoid and evict. */
   4391 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
   4392 		i915_gem_evict_everything(dev);
   4393 
   4394 	i915_kernel_lost_context(dev);
   4395 	i915_gem_cleanup_ringbuffer(dev);
   4396 
   4397 	/* Hack!  Don't let anybody do execbuf while we don't control the chip.
   4398 	 * We need to replace this with a semaphore, or something.
   4399 	 * And not confound ums.mm_suspended!
   4400 	 */
   4401 	dev_priv->ums.mm_suspended = !drm_core_check_feature(dev,
   4402 							     DRIVER_MODESET);
   4403 	mutex_unlock(&dev->struct_mutex);
   4404 
   4405 	del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
   4406 	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
   4407 	cancel_delayed_work_sync(&dev_priv->mm.idle_work);
   4408 
   4409 	return 0;
   4410 
   4411 err:
   4412 	mutex_unlock(&dev->struct_mutex);
   4413 	return ret;
   4414 }
   4415 
   4416 int i915_gem_l3_remap(struct intel_ring_buffer *ring, int slice)
   4417 {
   4418 	struct drm_device *dev = ring->dev;
   4419 	struct drm_i915_private *dev_priv = dev->dev_private;
   4420 	u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
   4421 	u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
   4422 	int i, ret;
   4423 
   4424 	if (!HAS_L3_DPF(dev) || !remap_info)
   4425 		return 0;
   4426 
   4427 	ret = intel_ring_begin(ring, GEN7_L3LOG_SIZE / 4 * 3);
   4428 	if (ret)
   4429 		return ret;
   4430 
   4431 	/*
   4432 	 * Note: We do not worry about the concurrent register cacheline hang
   4433 	 * here because no other code should access these registers other than
   4434 	 * at initialization time.
   4435 	 */
   4436 	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
   4437 		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
   4438 		intel_ring_emit(ring, reg_base + i);
   4439 		intel_ring_emit(ring, remap_info[i/4]);
   4440 	}
   4441 
   4442 	intel_ring_advance(ring);
   4443 
   4444 	return ret;
   4445 }
   4446 
   4447 void i915_gem_init_swizzling(struct drm_device *dev)
   4448 {
   4449 	struct drm_i915_private *dev_priv = dev->dev_private;
   4450 
   4451 	if (INTEL_INFO(dev)->gen < 5 ||
   4452 	    dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
   4453 		return;
   4454 
   4455 	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
   4456 				 DISP_TILE_SURFACE_SWIZZLING);
   4457 
   4458 	if (IS_GEN5(dev))
   4459 		return;
   4460 
   4461 	I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
   4462 	if (IS_GEN6(dev))
   4463 		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
   4464 	else if (IS_GEN7(dev))
   4465 		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
   4466 	else if (IS_GEN8(dev))
   4467 		I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
   4468 	else
   4469 		BUG();
   4470 }
   4471 
   4472 static bool
   4473 intel_enable_blt(struct drm_device *dev)
   4474 {
   4475 	if (!HAS_BLT(dev))
   4476 		return false;
   4477 
   4478 	/* The blitter was dysfunctional on early prototypes */
   4479 	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
   4480 		DRM_INFO("BLT not supported on this pre-production hardware;"
   4481 			 " graphics performance will be degraded.\n");
   4482 		return false;
   4483 	}
   4484 
   4485 	return true;
   4486 }
   4487 
   4488 static int i915_gem_init_rings(struct drm_device *dev)
   4489 {
   4490 	struct drm_i915_private *dev_priv = dev->dev_private;
   4491 	int ret;
   4492 
   4493 	ret = intel_init_render_ring_buffer(dev);
   4494 	if (ret)
   4495 		return ret;
   4496 
   4497 	if (HAS_BSD(dev)) {
   4498 		ret = intel_init_bsd_ring_buffer(dev);
   4499 		if (ret)
   4500 			goto cleanup_render_ring;
   4501 	}
   4502 
   4503 	if (intel_enable_blt(dev)) {
   4504 		ret = intel_init_blt_ring_buffer(dev);
   4505 		if (ret)
   4506 			goto cleanup_bsd_ring;
   4507 	}
   4508 
   4509 	if (HAS_VEBOX(dev)) {
   4510 		ret = intel_init_vebox_ring_buffer(dev);
   4511 		if (ret)
   4512 			goto cleanup_blt_ring;
   4513 	}
   4514 
   4515 
   4516 	ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
   4517 	if (ret)
   4518 		goto cleanup_vebox_ring;
   4519 
   4520 	return 0;
   4521 
   4522 cleanup_vebox_ring:
   4523 	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
   4524 cleanup_blt_ring:
   4525 	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
   4526 cleanup_bsd_ring:
   4527 	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
   4528 cleanup_render_ring:
   4529 	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
   4530 
   4531 	return ret;
   4532 }
   4533 
   4534 int
   4535 i915_gem_init_hw(struct drm_device *dev)
   4536 {
   4537 	struct drm_i915_private *dev_priv = dev->dev_private;
   4538 	int ret, i;
   4539 
   4540 	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
   4541 		return -EIO;
   4542 
   4543 	if (dev_priv->ellc_size)
   4544 		I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
   4545 
   4546 	if (IS_HASWELL(dev))
   4547 		I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
   4548 			   LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
   4549 
   4550 	if (HAS_PCH_NOP(dev)) {
   4551 		if (IS_IVYBRIDGE(dev)) {
   4552 			u32 temp = I915_READ(GEN7_MSG_CTL);
   4553 			temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
   4554 			I915_WRITE(GEN7_MSG_CTL, temp);
   4555 		} else if (INTEL_INFO(dev)->gen >= 7) {
   4556 			u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
   4557 			temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
   4558 			I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
   4559 		}
   4560 	}
   4561 
   4562 	i915_gem_init_swizzling(dev);
   4563 
   4564 	ret = i915_gem_init_rings(dev);
   4565 	if (ret)
   4566 		return ret;
   4567 
   4568 	for (i = 0; i < NUM_L3_SLICES(dev); i++)
   4569 		i915_gem_l3_remap(&dev_priv->ring[RCS], i);
   4570 
   4571 	/*
   4572 	 * XXX: Contexts should only be initialized once. Doing a switch to the
   4573 	 * default context switch however is something we'd like to do after
   4574 	 * reset or thaw (the latter may not actually be necessary for HW, but
   4575 	 * goes with our code better). Context switching requires rings (for
   4576 	 * the do_switch), but before enabling PPGTT. So don't move this.
   4577 	 */
   4578 	ret = i915_gem_context_enable(dev_priv);
   4579 	if (ret) {
   4580 		DRM_ERROR("Context enable failed %d\n", ret);
   4581 		goto err_out;
   4582 	}
   4583 
   4584 	return 0;
   4585 
   4586 err_out:
   4587 	i915_gem_cleanup_ringbuffer(dev);
   4588 	return ret;
   4589 }
   4590 
   4591 int i915_gem_init(struct drm_device *dev)
   4592 {
   4593 	struct drm_i915_private *dev_priv = dev->dev_private;
   4594 	int ret;
   4595 
   4596 	mutex_lock(&dev->struct_mutex);
   4597 
   4598 	if (IS_VALLEYVIEW(dev)) {
   4599 		/* VLVA0 (potential hack), BIOS isn't actually waking us */
   4600 		I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
   4601 		if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
   4602 			DRM_DEBUG_DRIVER("allow wake ack timed out\n");
   4603 	}
   4604 
   4605 	i915_gem_init_global_gtt(dev);
   4606 
   4607 	ret = i915_gem_context_init(dev);
   4608 	if (ret) {
   4609 		mutex_unlock(&dev->struct_mutex);
   4610 		return ret;
   4611 	}
   4612 
   4613 	ret = i915_gem_init_hw(dev);
   4614 	mutex_unlock(&dev->struct_mutex);
   4615 	if (ret) {
   4616 		WARN_ON(dev_priv->mm.aliasing_ppgtt);
   4617 		i915_gem_context_fini(dev);
   4618 		drm_mm_takedown(&dev_priv->gtt.base.mm);
   4619 		return ret;
   4620 	}
   4621 
   4622 	/* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
   4623 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
   4624 		dev_priv->dri1.allow_batchbuffer = 1;
   4625 	return 0;
   4626 }
   4627 
   4628 void
   4629 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
   4630 {
   4631 	struct drm_i915_private *dev_priv = dev->dev_private;
   4632 	struct intel_ring_buffer *ring;
   4633 	int i;
   4634 
   4635 	for_each_ring(ring, dev_priv, i)
   4636 		intel_cleanup_ring_buffer(ring);
   4637 }
   4638 
   4639 int
   4640 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
   4641 		       struct drm_file *file_priv)
   4642 {
   4643 	struct drm_i915_private *dev_priv = dev->dev_private;
   4644 	int ret;
   4645 
   4646 	if (drm_core_check_feature(dev, DRIVER_MODESET))
   4647 		return 0;
   4648 
   4649 	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
   4650 		DRM_ERROR("Reenabling wedged hardware, good luck\n");
   4651 		atomic_set(&dev_priv->gpu_error.reset_counter, 0);
   4652 	}
   4653 
   4654 	mutex_lock(&dev->struct_mutex);
   4655 	dev_priv->ums.mm_suspended = 0;
   4656 
   4657 	ret = i915_gem_init_hw(dev);
   4658 	if (ret != 0) {
   4659 		mutex_unlock(&dev->struct_mutex);
   4660 		return ret;
   4661 	}
   4662 
   4663 	BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
   4664 	mutex_unlock(&dev->struct_mutex);
   4665 
   4666 	ret = drm_irq_install(dev);
   4667 	if (ret)
   4668 		goto cleanup_ringbuffer;
   4669 
   4670 	return 0;
   4671 
   4672 cleanup_ringbuffer:
   4673 	mutex_lock(&dev->struct_mutex);
   4674 	i915_gem_cleanup_ringbuffer(dev);
   4675 	dev_priv->ums.mm_suspended = 1;
   4676 	mutex_unlock(&dev->struct_mutex);
   4677 
   4678 	return ret;
   4679 }
   4680 
   4681 int
   4682 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
   4683 		       struct drm_file *file_priv)
   4684 {
   4685 	if (drm_core_check_feature(dev, DRIVER_MODESET))
   4686 		return 0;
   4687 
   4688 	drm_irq_uninstall(dev);
   4689 
   4690 	return i915_gem_suspend(dev);
   4691 }
   4692 
   4693 void
   4694 i915_gem_lastclose(struct drm_device *dev)
   4695 {
   4696 	int ret;
   4697 
   4698 	if (drm_core_check_feature(dev, DRIVER_MODESET))
   4699 		return;
   4700 
   4701 	ret = i915_gem_suspend(dev);
   4702 	if (ret)
   4703 		DRM_ERROR("failed to idle hardware: %d\n", ret);
   4704 }
   4705 
   4706 static void
   4707 init_ring_lists(struct intel_ring_buffer *ring)
   4708 {
   4709 	INIT_LIST_HEAD(&ring->active_list);
   4710 	INIT_LIST_HEAD(&ring->request_list);
   4711 }
   4712 
   4713 void i915_init_vm(struct drm_i915_private *dev_priv,
   4714 		  struct i915_address_space *vm)
   4715 {
   4716 	if (!i915_is_ggtt(vm))
   4717 		drm_mm_init(&vm->mm, vm->start, vm->total);
   4718 	vm->dev = dev_priv->dev;
   4719 	INIT_LIST_HEAD(&vm->active_list);
   4720 	INIT_LIST_HEAD(&vm->inactive_list);
   4721 	INIT_LIST_HEAD(&vm->global_link);
   4722 	list_add_tail(&vm->global_link, &dev_priv->vm_list);
   4723 }
   4724 
   4725 void
   4726 i915_gem_load(struct drm_device *dev)
   4727 {
   4728 	struct drm_i915_private *dev_priv = dev->dev_private;
   4729 	int i;
   4730 
   4731 	dev_priv->slab =
   4732 		kmem_cache_create("i915_gem_object",
   4733 				  sizeof(struct drm_i915_gem_object), 0,
   4734 				  SLAB_HWCACHE_ALIGN,
   4735 				  NULL);
   4736 
   4737 	INIT_LIST_HEAD(&dev_priv->vm_list);
   4738 	i915_init_vm(dev_priv, &dev_priv->gtt.base);
   4739 
   4740 	INIT_LIST_HEAD(&dev_priv->context_list);
   4741 	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
   4742 	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
   4743 	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
   4744 	for (i = 0; i < I915_NUM_RINGS; i++)
   4745 		init_ring_lists(&dev_priv->ring[i]);
   4746 	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
   4747 		INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
   4748 	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
   4749 			  i915_gem_retire_work_handler);
   4750 	INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
   4751 			  i915_gem_idle_work_handler);
   4752 	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
   4753 
   4754 	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
   4755 	if (IS_GEN3(dev)) {
   4756 		I915_WRITE(MI_ARB_STATE,
   4757 			   _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
   4758 	}
   4759 
   4760 	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
   4761 
   4762 	/* Old X drivers will take 0-2 for front, back, depth buffers */
   4763 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
   4764 		dev_priv->fence_reg_start = 3;
   4765 
   4766 	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
   4767 		dev_priv->num_fence_regs = 32;
   4768 	else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
   4769 		dev_priv->num_fence_regs = 16;
   4770 	else
   4771 		dev_priv->num_fence_regs = 8;
   4772 
   4773 	/* Initialize fence registers to zero */
   4774 	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
   4775 	i915_gem_restore_fences(dev);
   4776 
   4777 	i915_gem_detect_bit_6_swizzle(dev);
   4778 	init_waitqueue_head(&dev_priv->pending_flip_queue);
   4779 
   4780 	dev_priv->mm.interruptible = true;
   4781 
   4782 	dev_priv->mm.inactive_shrinker.scan_objects = i915_gem_inactive_scan;
   4783 	dev_priv->mm.inactive_shrinker.count_objects = i915_gem_inactive_count;
   4784 	dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
   4785 	register_shrinker(&dev_priv->mm.inactive_shrinker);
   4786 }
   4787 
   4788 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
   4789 {
   4790 	struct drm_i915_file_private *file_priv = file->driver_priv;
   4791 
   4792 	cancel_delayed_work_sync(&file_priv->mm.idle_work);
   4793 
   4794 	/* Clean up our request list when the client is going away, so that
   4795 	 * later retire_requests won't dereference our soon-to-be-gone
   4796 	 * file_priv.
   4797 	 */
   4798 	spin_lock(&file_priv->mm.lock);
   4799 	while (!list_empty(&file_priv->mm.request_list)) {
   4800 		struct drm_i915_gem_request *request;
   4801 
   4802 		request = list_first_entry(&file_priv->mm.request_list,
   4803 					   struct drm_i915_gem_request,
   4804 					   client_list);
   4805 		list_del(&request->client_list);
   4806 		request->file_priv = NULL;
   4807 	}
   4808 	spin_unlock(&file_priv->mm.lock);
   4809 }
   4810 
   4811 static void
   4812 i915_gem_file_idle_work_handler(struct work_struct *work)
   4813 {
   4814 	struct drm_i915_file_private *file_priv =
   4815 		container_of(work, typeof(*file_priv), mm.idle_work.work);
   4816 
   4817 	atomic_set(&file_priv->rps_wait_boost, false);
   4818 }
   4819 
   4820 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
   4821 {
   4822 	struct drm_i915_file_private *file_priv;
   4823 	int ret;
   4824 
   4825 	DRM_DEBUG_DRIVER("\n");
   4826 
   4827 	file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
   4828 	if (!file_priv)
   4829 		return -ENOMEM;
   4830 
   4831 	file->driver_priv = file_priv;
   4832 	file_priv->dev_priv = dev->dev_private;
   4833 	file_priv->file = file;
   4834 
   4835 	spin_lock_init(&file_priv->mm.lock);
   4836 	INIT_LIST_HEAD(&file_priv->mm.request_list);
   4837 	INIT_DELAYED_WORK(&file_priv->mm.idle_work,
   4838 			  i915_gem_file_idle_work_handler);
   4839 
   4840 	ret = i915_gem_context_open(dev, file);
   4841 	if (ret)
   4842 		kfree(file_priv);
   4843 
   4844 	return ret;
   4845 }
   4846 
   4847 static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
   4848 {
   4849 	if (!mutex_is_locked(mutex))
   4850 		return false;
   4851 
   4852 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
   4853 	return mutex->owner == task;
   4854 #else
   4855 	/* Since UP may be pre-empted, we cannot assume that we own the lock */
   4856 	return false;
   4857 #endif
   4858 }
   4859 
   4860 static unsigned long
   4861 i915_gem_inactive_count(struct shrinker *shrinker, struct shrink_control *sc)
   4862 {
   4863 	struct drm_i915_private *dev_priv =
   4864 		container_of(shrinker,
   4865 			     struct drm_i915_private,
   4866 			     mm.inactive_shrinker);
   4867 	struct drm_device *dev = dev_priv->dev;
   4868 	struct drm_i915_gem_object *obj;
   4869 	bool unlock = true;
   4870 	unsigned long count;
   4871 
   4872 	if (!mutex_trylock(&dev->struct_mutex)) {
   4873 		if (!mutex_is_locked_by(&dev->struct_mutex, current))
   4874 			return 0;
   4875 
   4876 		if (dev_priv->mm.shrinker_no_lock_stealing)
   4877 			return 0;
   4878 
   4879 		unlock = false;
   4880 	}
   4881 
   4882 	count = 0;
   4883 	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
   4884 		if (obj->pages_pin_count == 0)
   4885 			count += obj->base.size >> PAGE_SHIFT;
   4886 
   4887 	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
   4888 		if (obj->active)
   4889 			continue;
   4890 
   4891 		if (!i915_gem_obj_is_pinned(obj) && obj->pages_pin_count == 0)
   4892 			count += obj->base.size >> PAGE_SHIFT;
   4893 	}
   4894 
   4895 	if (unlock)
   4896 		mutex_unlock(&dev->struct_mutex);
   4897 
   4898 	return count;
   4899 }
   4900 
   4901 /* All the new VM stuff */
   4902 unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
   4903 				  struct i915_address_space *vm)
   4904 {
   4905 	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
   4906 	struct i915_vma *vma;
   4907 
   4908 	if (!dev_priv->mm.aliasing_ppgtt ||
   4909 	    vm == &dev_priv->mm.aliasing_ppgtt->base)
   4910 		vm = &dev_priv->gtt.base;
   4911 
   4912 	BUG_ON(list_empty(&o->vma_list));
   4913 	list_for_each_entry(vma, &o->vma_list, vma_link) {
   4914 		if (vma->vm == vm)
   4915 			return vma->node.start;
   4916 
   4917 	}
   4918 	return -1;
   4919 }
   4920 
   4921 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
   4922 			struct i915_address_space *vm)
   4923 {
   4924 	struct i915_vma *vma;
   4925 
   4926 	list_for_each_entry(vma, &o->vma_list, vma_link)
   4927 		if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
   4928 			return true;
   4929 
   4930 	return false;
   4931 }
   4932 
   4933 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
   4934 {
   4935 	struct i915_vma *vma;
   4936 
   4937 	list_for_each_entry(vma, &o->vma_list, vma_link)
   4938 		if (drm_mm_node_allocated(&vma->node))
   4939 			return true;
   4940 
   4941 	return false;
   4942 }
   4943 
   4944 unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
   4945 				struct i915_address_space *vm)
   4946 {
   4947 	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
   4948 	struct i915_vma *vma;
   4949 
   4950 	if (!dev_priv->mm.aliasing_ppgtt ||
   4951 	    vm == &dev_priv->mm.aliasing_ppgtt->base)
   4952 		vm = &dev_priv->gtt.base;
   4953 
   4954 	BUG_ON(list_empty(&o->vma_list));
   4955 
   4956 	list_for_each_entry(vma, &o->vma_list, vma_link)
   4957 		if (vma->vm == vm)
   4958 			return vma->node.size;
   4959 
   4960 	return 0;
   4961 }
   4962 
   4963 static unsigned long
   4964 i915_gem_inactive_scan(struct shrinker *shrinker, struct shrink_control *sc)
   4965 {
   4966 	struct drm_i915_private *dev_priv =
   4967 		container_of(shrinker,
   4968 			     struct drm_i915_private,
   4969 			     mm.inactive_shrinker);
   4970 	struct drm_device *dev = dev_priv->dev;
   4971 	unsigned long freed;
   4972 	bool unlock = true;
   4973 
   4974 	if (!mutex_trylock(&dev->struct_mutex)) {
   4975 		if (!mutex_is_locked_by(&dev->struct_mutex, current))
   4976 			return SHRINK_STOP;
   4977 
   4978 		if (dev_priv->mm.shrinker_no_lock_stealing)
   4979 			return SHRINK_STOP;
   4980 
   4981 		unlock = false;
   4982 	}
   4983 
   4984 	freed = i915_gem_purge(dev_priv, sc->nr_to_scan);
   4985 	if (freed < sc->nr_to_scan)
   4986 		freed += __i915_gem_shrink(dev_priv,
   4987 					   sc->nr_to_scan - freed,
   4988 					   false);
   4989 	if (freed < sc->nr_to_scan)
   4990 		freed += i915_gem_shrink_all(dev_priv);
   4991 
   4992 	if (unlock)
   4993 		mutex_unlock(&dev->struct_mutex);
   4994 
   4995 	return freed;
   4996 }
   4997 
   4998 struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj)
   4999 {
   5000 	struct i915_vma *vma;
   5001 
   5002 	if (WARN_ON(list_empty(&obj->vma_list)))
   5003 		return NULL;
   5004 
   5005 	vma = list_first_entry(&obj->vma_list, typeof(*vma), vma_link);
   5006 	if (vma->vm != obj_to_ggtt(obj))
   5007 		return NULL;
   5008 
   5009 	return vma;
   5010 }
   5011