Home | History | Annotate | Line # | Download | only in i915
i915_drv.c revision 1.1.1.4
      1 /*	$NetBSD: i915_drv.c,v 1.1.1.4 2021/12/18 20:15:24 riastradh Exp $	*/
      2 
      3 /* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*-
      4  */
      5 /*
      6  *
      7  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
      8  * All Rights Reserved.
      9  *
     10  * Permission is hereby granted, free of charge, to any person obtaining a
     11  * copy of this software and associated documentation files (the
     12  * "Software"), to deal in the Software without restriction, including
     13  * without limitation the rights to use, copy, modify, merge, publish,
     14  * distribute, sub license, and/or sell copies of the Software, and to
     15  * permit persons to whom the Software is furnished to do so, subject to
     16  * the following conditions:
     17  *
     18  * The above copyright notice and this permission notice (including the
     19  * next paragraph) shall be included in all copies or substantial portions
     20  * of the Software.
     21  *
     22  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
     23  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
     24  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
     25  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
     26  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
     27  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
     28  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
     29  *
     30  */
     31 
     32 #include <sys/cdefs.h>
     33 __KERNEL_RCSID(0, "$NetBSD: i915_drv.c,v 1.1.1.4 2021/12/18 20:15:24 riastradh Exp $");
     34 
     35 #include <linux/acpi.h>
     36 #include <linux/device.h>
     37 #include <linux/oom.h>
     38 #include <linux/module.h>
     39 #include <linux/pci.h>
     40 #include <linux/pm.h>
     41 #include <linux/pm_runtime.h>
     42 #include <linux/pnp.h>
     43 #include <linux/slab.h>
     44 #include <linux/vga_switcheroo.h>
     45 #include <linux/vt.h>
     46 #include <acpi/video.h>
     47 
     48 #include <drm/drm_atomic_helper.h>
     49 #include <drm/drm_ioctl.h>
     50 #include <drm/drm_irq.h>
     51 #include <drm/drm_probe_helper.h>
     52 #include <drm/i915_drm.h>
     53 
     54 #include "display/intel_acpi.h"
     55 #include "display/intel_audio.h"
     56 #include "display/intel_bw.h"
     57 #include "display/intel_cdclk.h"
     58 #include "display/intel_display_types.h"
     59 #include "display/intel_dp.h"
     60 #include "display/intel_fbdev.h"
     61 #include "display/intel_hotplug.h"
     62 #include "display/intel_overlay.h"
     63 #include "display/intel_pipe_crc.h"
     64 #include "display/intel_sprite.h"
     65 #include "display/intel_vga.h"
     66 
     67 #include "gem/i915_gem_context.h"
     68 #include "gem/i915_gem_ioctls.h"
     69 #include "gem/i915_gem_mman.h"
     70 #include "gt/intel_gt.h"
     71 #include "gt/intel_gt_pm.h"
     72 #include "gt/intel_rc6.h"
     73 
     74 #include "i915_debugfs.h"
     75 #include "i915_drv.h"
     76 #include "i915_irq.h"
     77 #include "i915_memcpy.h"
     78 #include "i915_perf.h"
     79 #include "i915_query.h"
     80 #include "i915_suspend.h"
     81 #include "i915_switcheroo.h"
     82 #include "i915_sysfs.h"
     83 #include "i915_trace.h"
     84 #include "i915_vgpu.h"
     85 #include "intel_csr.h"
     86 #include "intel_memory_region.h"
     87 #include "intel_pm.h"
     88 
     89 static struct drm_driver driver;
     90 
     91 struct vlv_s0ix_state {
     92 	/* GAM */
     93 	u32 wr_watermark;
     94 	u32 gfx_prio_ctrl;
     95 	u32 arb_mode;
     96 	u32 gfx_pend_tlb0;
     97 	u32 gfx_pend_tlb1;
     98 	u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM];
     99 	u32 media_max_req_count;
    100 	u32 gfx_max_req_count;
    101 	u32 render_hwsp;
    102 	u32 ecochk;
    103 	u32 bsd_hwsp;
    104 	u32 blt_hwsp;
    105 	u32 tlb_rd_addr;
    106 
    107 	/* MBC */
    108 	u32 g3dctl;
    109 	u32 gsckgctl;
    110 	u32 mbctl;
    111 
    112 	/* GCP */
    113 	u32 ucgctl1;
    114 	u32 ucgctl3;
    115 	u32 rcgctl1;
    116 	u32 rcgctl2;
    117 	u32 rstctl;
    118 	u32 misccpctl;
    119 
    120 	/* GPM */
    121 	u32 gfxpause;
    122 	u32 rpdeuhwtc;
    123 	u32 rpdeuc;
    124 	u32 ecobus;
    125 	u32 pwrdwnupctl;
    126 	u32 rp_down_timeout;
    127 	u32 rp_deucsw;
    128 	u32 rcubmabdtmr;
    129 	u32 rcedata;
    130 	u32 spare2gh;
    131 
    132 	/* Display 1 CZ domain */
    133 	u32 gt_imr;
    134 	u32 gt_ier;
    135 	u32 pm_imr;
    136 	u32 pm_ier;
    137 	u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM];
    138 
    139 	/* GT SA CZ domain */
    140 	u32 tilectl;
    141 	u32 gt_fifoctl;
    142 	u32 gtlc_wake_ctrl;
    143 	u32 gtlc_survive;
    144 	u32 pmwgicz;
    145 
    146 	/* Display 2 CZ domain */
    147 	u32 gu_ctl0;
    148 	u32 gu_ctl1;
    149 	u32 pcbr;
    150 	u32 clock_gate_dis2;
    151 };
    152 
    153 static int i915_get_bridge_dev(struct drm_i915_private *dev_priv)
    154 {
    155 	int domain = pci_domain_nr(dev_priv->drm.pdev->bus);
    156 
    157 	dev_priv->bridge_dev =
    158 		pci_get_domain_bus_and_slot(domain, 0, PCI_DEVFN(0, 0));
    159 	if (!dev_priv->bridge_dev) {
    160 		DRM_ERROR("bridge device not found\n");
    161 		return -1;
    162 	}
    163 	return 0;
    164 }
    165 
    166 /* Allocate space for the MCH regs if needed, return nonzero on error */
    167 static int
    168 intel_alloc_mchbar_resource(struct drm_i915_private *dev_priv)
    169 {
    170 	int reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
    171 	u32 temp_lo, temp_hi = 0;
    172 	u64 mchbar_addr;
    173 	int ret;
    174 
    175 	if (INTEL_GEN(dev_priv) >= 4)
    176 		pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi);
    177 	pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo);
    178 	mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
    179 
    180 	/* If ACPI doesn't have it, assume we need to allocate it ourselves */
    181 #ifdef CONFIG_PNP
    182 	if (mchbar_addr &&
    183 	    pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE))
    184 		return 0;
    185 #endif
    186 
    187 	/* Get some space for it */
    188 	dev_priv->mch_res.name = "i915 MCHBAR";
    189 	dev_priv->mch_res.flags = IORESOURCE_MEM;
    190 	ret = pci_bus_alloc_resource(dev_priv->bridge_dev->bus,
    191 				     &dev_priv->mch_res,
    192 				     MCHBAR_SIZE, MCHBAR_SIZE,
    193 				     PCIBIOS_MIN_MEM,
    194 				     0, pcibios_align_resource,
    195 				     dev_priv->bridge_dev);
    196 	if (ret) {
    197 		DRM_DEBUG_DRIVER("failed bus alloc: %d\n", ret);
    198 		dev_priv->mch_res.start = 0;
    199 		return ret;
    200 	}
    201 
    202 	if (INTEL_GEN(dev_priv) >= 4)
    203 		pci_write_config_dword(dev_priv->bridge_dev, reg + 4,
    204 				       upper_32_bits(dev_priv->mch_res.start));
    205 
    206 	pci_write_config_dword(dev_priv->bridge_dev, reg,
    207 			       lower_32_bits(dev_priv->mch_res.start));
    208 	return 0;
    209 }
    210 
    211 /* Setup MCHBAR if possible, return true if we should disable it again */
    212 static void
    213 intel_setup_mchbar(struct drm_i915_private *dev_priv)
    214 {
    215 	int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
    216 	u32 temp;
    217 	bool enabled;
    218 
    219 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
    220 		return;
    221 
    222 	dev_priv->mchbar_need_disable = false;
    223 
    224 	if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
    225 		pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp);
    226 		enabled = !!(temp & DEVEN_MCHBAR_EN);
    227 	} else {
    228 		pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
    229 		enabled = temp & 1;
    230 	}
    231 
    232 	/* If it's already enabled, don't have to do anything */
    233 	if (enabled)
    234 		return;
    235 
    236 	if (intel_alloc_mchbar_resource(dev_priv))
    237 		return;
    238 
    239 	dev_priv->mchbar_need_disable = true;
    240 
    241 	/* Space is allocated or reserved, so enable it. */
    242 	if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
    243 		pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
    244 				       temp | DEVEN_MCHBAR_EN);
    245 	} else {
    246 		pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
    247 		pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1);
    248 	}
    249 }
    250 
    251 static void
    252 intel_teardown_mchbar(struct drm_i915_private *dev_priv)
    253 {
    254 	int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
    255 
    256 	if (dev_priv->mchbar_need_disable) {
    257 		if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
    258 			u32 deven_val;
    259 
    260 			pci_read_config_dword(dev_priv->bridge_dev, DEVEN,
    261 					      &deven_val);
    262 			deven_val &= ~DEVEN_MCHBAR_EN;
    263 			pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
    264 					       deven_val);
    265 		} else {
    266 			u32 mchbar_val;
    267 
    268 			pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg,
    269 					      &mchbar_val);
    270 			mchbar_val &= ~1;
    271 			pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg,
    272 					       mchbar_val);
    273 		}
    274 	}
    275 
    276 	if (dev_priv->mch_res.start)
    277 		release_resource(&dev_priv->mch_res);
    278 }
    279 
    280 static int i915_driver_modeset_probe(struct drm_i915_private *i915)
    281 {
    282 	int ret;
    283 
    284 	if (i915_inject_probe_failure(i915))
    285 		return -ENODEV;
    286 
    287 	if (HAS_DISPLAY(i915) && INTEL_DISPLAY_ENABLED(i915)) {
    288 		ret = drm_vblank_init(&i915->drm,
    289 				      INTEL_NUM_PIPES(i915));
    290 		if (ret)
    291 			goto out;
    292 	}
    293 
    294 	intel_bios_init(i915);
    295 
    296 	ret = intel_vga_register(i915);
    297 	if (ret)
    298 		goto out;
    299 
    300 	intel_register_dsm_handler();
    301 
    302 	ret = i915_switcheroo_register(i915);
    303 	if (ret)
    304 		goto cleanup_vga_client;
    305 
    306 	intel_power_domains_init_hw(i915, false);
    307 
    308 	intel_csr_ucode_init(i915);
    309 
    310 	ret = intel_irq_install(i915);
    311 	if (ret)
    312 		goto cleanup_csr;
    313 
    314 	/* Important: The output setup functions called by modeset_init need
    315 	 * working irqs for e.g. gmbus and dp aux transfers. */
    316 	ret = intel_modeset_init(i915);
    317 	if (ret)
    318 		goto cleanup_irq;
    319 
    320 	ret = i915_gem_init(i915);
    321 	if (ret)
    322 		goto cleanup_modeset;
    323 
    324 	intel_overlay_setup(i915);
    325 
    326 	if (!HAS_DISPLAY(i915) || !INTEL_DISPLAY_ENABLED(i915))
    327 		return 0;
    328 
    329 	ret = intel_fbdev_init(&i915->drm);
    330 	if (ret)
    331 		goto cleanup_gem;
    332 
    333 	/* Only enable hotplug handling once the fbdev is fully set up. */
    334 	intel_hpd_init(i915);
    335 
    336 	intel_init_ipc(i915);
    337 
    338 	return 0;
    339 
    340 cleanup_gem:
    341 	i915_gem_suspend(i915);
    342 	i915_gem_driver_remove(i915);
    343 	i915_gem_driver_release(i915);
    344 cleanup_modeset:
    345 	intel_modeset_driver_remove(i915);
    346 cleanup_irq:
    347 	intel_irq_uninstall(i915);
    348 cleanup_csr:
    349 	intel_csr_ucode_fini(i915);
    350 	intel_power_domains_driver_remove(i915);
    351 	i915_switcheroo_unregister(i915);
    352 cleanup_vga_client:
    353 	intel_vga_unregister(i915);
    354 out:
    355 	return ret;
    356 }
    357 
    358 static void i915_driver_modeset_remove(struct drm_i915_private *i915)
    359 {
    360 	intel_modeset_driver_remove(i915);
    361 
    362 	intel_irq_uninstall(i915);
    363 
    364 	intel_bios_driver_remove(i915);
    365 
    366 	i915_switcheroo_unregister(i915);
    367 
    368 	intel_vga_unregister(i915);
    369 
    370 	intel_csr_ucode_fini(i915);
    371 }
    372 
    373 static void intel_init_dpio(struct drm_i915_private *dev_priv)
    374 {
    375 	/*
    376 	 * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C),
    377 	 * CHV x1 PHY (DP/HDMI D)
    378 	 * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C)
    379 	 */
    380 	if (IS_CHERRYVIEW(dev_priv)) {
    381 		DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2;
    382 		DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO;
    383 	} else if (IS_VALLEYVIEW(dev_priv)) {
    384 		DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO;
    385 	}
    386 }
    387 
    388 static int i915_workqueues_init(struct drm_i915_private *dev_priv)
    389 {
    390 	/*
    391 	 * The i915 workqueue is primarily used for batched retirement of
    392 	 * requests (and thus managing bo) once the task has been completed
    393 	 * by the GPU. i915_retire_requests() is called directly when we
    394 	 * need high-priority retirement, such as waiting for an explicit
    395 	 * bo.
    396 	 *
    397 	 * It is also used for periodic low-priority events, such as
    398 	 * idle-timers and recording error state.
    399 	 *
    400 	 * All tasks on the workqueue are expected to acquire the dev mutex
    401 	 * so there is no point in running more than one instance of the
    402 	 * workqueue at any time.  Use an ordered one.
    403 	 */
    404 	dev_priv->wq = alloc_ordered_workqueue("i915", 0);
    405 	if (dev_priv->wq == NULL)
    406 		goto out_err;
    407 
    408 	dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0);
    409 	if (dev_priv->hotplug.dp_wq == NULL)
    410 		goto out_free_wq;
    411 
    412 	return 0;
    413 
    414 out_free_wq:
    415 	destroy_workqueue(dev_priv->wq);
    416 out_err:
    417 	DRM_ERROR("Failed to allocate workqueues.\n");
    418 
    419 	return -ENOMEM;
    420 }
    421 
    422 static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
    423 {
    424 	destroy_workqueue(dev_priv->hotplug.dp_wq);
    425 	destroy_workqueue(dev_priv->wq);
    426 }
    427 
    428 /*
    429  * We don't keep the workarounds for pre-production hardware, so we expect our
    430  * driver to fail on these machines in one way or another. A little warning on
    431  * dmesg may help both the user and the bug triagers.
    432  *
    433  * Our policy for removing pre-production workarounds is to keep the
    434  * current gen workarounds as a guide to the bring-up of the next gen
    435  * (workarounds have a habit of persisting!). Anything older than that
    436  * should be removed along with the complications they introduce.
    437  */
    438 static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv)
    439 {
    440 	bool pre = false;
    441 
    442 	pre |= IS_HSW_EARLY_SDV(dev_priv);
    443 	pre |= IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0);
    444 	pre |= IS_BXT_REVID(dev_priv, 0, BXT_REVID_B_LAST);
    445 	pre |= IS_KBL_REVID(dev_priv, 0, KBL_REVID_A0);
    446 
    447 	if (pre) {
    448 		DRM_ERROR("This is a pre-production stepping. "
    449 			  "It may not be fully functional.\n");
    450 		add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK);
    451 	}
    452 }
    453 
    454 static int vlv_alloc_s0ix_state(struct drm_i915_private *i915)
    455 {
    456 	if (!IS_VALLEYVIEW(i915))
    457 		return 0;
    458 
    459 	/* we write all the values in the struct, so no need to zero it out */
    460 	i915->vlv_s0ix_state = kmalloc(sizeof(*i915->vlv_s0ix_state),
    461 				       GFP_KERNEL);
    462 	if (!i915->vlv_s0ix_state)
    463 		return -ENOMEM;
    464 
    465 	return 0;
    466 }
    467 
    468 static void vlv_free_s0ix_state(struct drm_i915_private *i915)
    469 {
    470 	if (!i915->vlv_s0ix_state)
    471 		return;
    472 
    473 	kfree(i915->vlv_s0ix_state);
    474 	i915->vlv_s0ix_state = NULL;
    475 }
    476 
    477 static void sanitize_gpu(struct drm_i915_private *i915)
    478 {
    479 	if (!INTEL_INFO(i915)->gpu_reset_clobbers_display)
    480 		__intel_gt_reset(&i915->gt, ALL_ENGINES);
    481 }
    482 
    483 /**
    484  * i915_driver_early_probe - setup state not requiring device access
    485  * @dev_priv: device private
    486  *
    487  * Initialize everything that is a "SW-only" state, that is state not
    488  * requiring accessing the device or exposing the driver via kernel internal
    489  * or userspace interfaces. Example steps belonging here: lock initialization,
    490  * system memory allocation, setting up device specific attributes and
    491  * function hooks not requiring accessing the device.
    492  */
    493 static int i915_driver_early_probe(struct drm_i915_private *dev_priv)
    494 {
    495 	int ret = 0;
    496 
    497 	if (i915_inject_probe_failure(dev_priv))
    498 		return -ENODEV;
    499 
    500 	intel_device_info_subplatform_init(dev_priv);
    501 
    502 	intel_uncore_mmio_debug_init_early(&dev_priv->mmio_debug);
    503 	intel_uncore_init_early(&dev_priv->uncore, dev_priv);
    504 
    505 	spin_lock_init(&dev_priv->irq_lock);
    506 	spin_lock_init(&dev_priv->gpu_error.lock);
    507 	mutex_init(&dev_priv->backlight_lock);
    508 
    509 	mutex_init(&dev_priv->sb_lock);
    510 	pm_qos_add_request(&dev_priv->sb_qos,
    511 			   PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
    512 
    513 	mutex_init(&dev_priv->av_mutex);
    514 	mutex_init(&dev_priv->wm.wm_mutex);
    515 	mutex_init(&dev_priv->pps_mutex);
    516 	mutex_init(&dev_priv->hdcp_comp_mutex);
    517 
    518 	i915_memcpy_init_early(dev_priv);
    519 	intel_runtime_pm_init_early(&dev_priv->runtime_pm);
    520 
    521 	ret = i915_workqueues_init(dev_priv);
    522 	if (ret < 0)
    523 		return ret;
    524 
    525 	ret = vlv_alloc_s0ix_state(dev_priv);
    526 	if (ret < 0)
    527 		goto err_workqueues;
    528 
    529 	intel_wopcm_init_early(&dev_priv->wopcm);
    530 
    531 	intel_gt_init_early(&dev_priv->gt, dev_priv);
    532 
    533 	i915_gem_init_early(dev_priv);
    534 
    535 	/* This must be called before any calls to HAS_PCH_* */
    536 	intel_detect_pch(dev_priv);
    537 
    538 	intel_pm_setup(dev_priv);
    539 	intel_init_dpio(dev_priv);
    540 	ret = intel_power_domains_init(dev_priv);
    541 	if (ret < 0)
    542 		goto err_gem;
    543 	intel_irq_init(dev_priv);
    544 	intel_init_display_hooks(dev_priv);
    545 	intel_init_clock_gating_hooks(dev_priv);
    546 	intel_init_audio_hooks(dev_priv);
    547 	intel_display_crc_init(dev_priv);
    548 
    549 	intel_detect_preproduction_hw(dev_priv);
    550 
    551 	return 0;
    552 
    553 err_gem:
    554 	i915_gem_cleanup_early(dev_priv);
    555 	intel_gt_driver_late_release(&dev_priv->gt);
    556 	vlv_free_s0ix_state(dev_priv);
    557 err_workqueues:
    558 	i915_workqueues_cleanup(dev_priv);
    559 	return ret;
    560 }
    561 
    562 /**
    563  * i915_driver_late_release - cleanup the setup done in
    564  *			       i915_driver_early_probe()
    565  * @dev_priv: device private
    566  */
    567 static void i915_driver_late_release(struct drm_i915_private *dev_priv)
    568 {
    569 	intel_irq_fini(dev_priv);
    570 	intel_power_domains_cleanup(dev_priv);
    571 	i915_gem_cleanup_early(dev_priv);
    572 	intel_gt_driver_late_release(&dev_priv->gt);
    573 	vlv_free_s0ix_state(dev_priv);
    574 	i915_workqueues_cleanup(dev_priv);
    575 
    576 	pm_qos_remove_request(&dev_priv->sb_qos);
    577 	mutex_destroy(&dev_priv->sb_lock);
    578 }
    579 
    580 /**
    581  * i915_driver_mmio_probe - setup device MMIO
    582  * @dev_priv: device private
    583  *
    584  * Setup minimal device state necessary for MMIO accesses later in the
    585  * initialization sequence. The setup here should avoid any other device-wide
    586  * side effects or exposing the driver via kernel internal or user space
    587  * interfaces.
    588  */
    589 static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv)
    590 {
    591 	int ret;
    592 
    593 	if (i915_inject_probe_failure(dev_priv))
    594 		return -ENODEV;
    595 
    596 	if (i915_get_bridge_dev(dev_priv))
    597 		return -EIO;
    598 
    599 	ret = intel_uncore_init_mmio(&dev_priv->uncore);
    600 	if (ret < 0)
    601 		goto err_bridge;
    602 
    603 	/* Try to make sure MCHBAR is enabled before poking at it */
    604 	intel_setup_mchbar(dev_priv);
    605 
    606 	intel_device_info_init_mmio(dev_priv);
    607 
    608 	intel_uncore_prune_mmio_domains(&dev_priv->uncore);
    609 
    610 	intel_uc_init_mmio(&dev_priv->gt.uc);
    611 
    612 	ret = intel_engines_init_mmio(&dev_priv->gt);
    613 	if (ret)
    614 		goto err_uncore;
    615 
    616 	/* As early as possible, scrub existing GPU state before clobbering */
    617 	sanitize_gpu(dev_priv);
    618 
    619 	return 0;
    620 
    621 err_uncore:
    622 	intel_teardown_mchbar(dev_priv);
    623 	intel_uncore_fini_mmio(&dev_priv->uncore);
    624 err_bridge:
    625 	pci_dev_put(dev_priv->bridge_dev);
    626 
    627 	return ret;
    628 }
    629 
    630 /**
    631  * i915_driver_mmio_release - cleanup the setup done in i915_driver_mmio_probe()
    632  * @dev_priv: device private
    633  */
    634 static void i915_driver_mmio_release(struct drm_i915_private *dev_priv)
    635 {
    636 	intel_teardown_mchbar(dev_priv);
    637 	intel_uncore_fini_mmio(&dev_priv->uncore);
    638 	pci_dev_put(dev_priv->bridge_dev);
    639 }
    640 
    641 static void intel_sanitize_options(struct drm_i915_private *dev_priv)
    642 {
    643 	intel_gvt_sanitize_options(dev_priv);
    644 }
    645 
    646 #define DRAM_TYPE_STR(type) [INTEL_DRAM_ ## type] = #type
    647 
    648 static const char *intel_dram_type_str(enum intel_dram_type type)
    649 {
    650 	static const char * const str[] = {
    651 		DRAM_TYPE_STR(UNKNOWN),
    652 		DRAM_TYPE_STR(DDR3),
    653 		DRAM_TYPE_STR(DDR4),
    654 		DRAM_TYPE_STR(LPDDR3),
    655 		DRAM_TYPE_STR(LPDDR4),
    656 	};
    657 
    658 	if (type >= ARRAY_SIZE(str))
    659 		type = INTEL_DRAM_UNKNOWN;
    660 
    661 	return str[type];
    662 }
    663 
    664 #undef DRAM_TYPE_STR
    665 
    666 static int intel_dimm_num_devices(const struct dram_dimm_info *dimm)
    667 {
    668 	return dimm->ranks * 64 / (dimm->width ?: 1);
    669 }
    670 
    671 /* Returns total GB for the whole DIMM */
    672 static int skl_get_dimm_size(u16 val)
    673 {
    674 	return val & SKL_DRAM_SIZE_MASK;
    675 }
    676 
    677 static int skl_get_dimm_width(u16 val)
    678 {
    679 	if (skl_get_dimm_size(val) == 0)
    680 		return 0;
    681 
    682 	switch (val & SKL_DRAM_WIDTH_MASK) {
    683 	case SKL_DRAM_WIDTH_X8:
    684 	case SKL_DRAM_WIDTH_X16:
    685 	case SKL_DRAM_WIDTH_X32:
    686 		val = (val & SKL_DRAM_WIDTH_MASK) >> SKL_DRAM_WIDTH_SHIFT;
    687 		return 8 << val;
    688 	default:
    689 		MISSING_CASE(val);
    690 		return 0;
    691 	}
    692 }
    693 
    694 static int skl_get_dimm_ranks(u16 val)
    695 {
    696 	if (skl_get_dimm_size(val) == 0)
    697 		return 0;
    698 
    699 	val = (val & SKL_DRAM_RANK_MASK) >> SKL_DRAM_RANK_SHIFT;
    700 
    701 	return val + 1;
    702 }
    703 
    704 /* Returns total GB for the whole DIMM */
    705 static int cnl_get_dimm_size(u16 val)
    706 {
    707 	return (val & CNL_DRAM_SIZE_MASK) / 2;
    708 }
    709 
    710 static int cnl_get_dimm_width(u16 val)
    711 {
    712 	if (cnl_get_dimm_size(val) == 0)
    713 		return 0;
    714 
    715 	switch (val & CNL_DRAM_WIDTH_MASK) {
    716 	case CNL_DRAM_WIDTH_X8:
    717 	case CNL_DRAM_WIDTH_X16:
    718 	case CNL_DRAM_WIDTH_X32:
    719 		val = (val & CNL_DRAM_WIDTH_MASK) >> CNL_DRAM_WIDTH_SHIFT;
    720 		return 8 << val;
    721 	default:
    722 		MISSING_CASE(val);
    723 		return 0;
    724 	}
    725 }
    726 
    727 static int cnl_get_dimm_ranks(u16 val)
    728 {
    729 	if (cnl_get_dimm_size(val) == 0)
    730 		return 0;
    731 
    732 	val = (val & CNL_DRAM_RANK_MASK) >> CNL_DRAM_RANK_SHIFT;
    733 
    734 	return val + 1;
    735 }
    736 
    737 static bool
    738 skl_is_16gb_dimm(const struct dram_dimm_info *dimm)
    739 {
    740 	/* Convert total GB to Gb per DRAM device */
    741 	return 8 * dimm->size / (intel_dimm_num_devices(dimm) ?: 1) == 16;
    742 }
    743 
    744 static void
    745 skl_dram_get_dimm_info(struct drm_i915_private *dev_priv,
    746 		       struct dram_dimm_info *dimm,
    747 		       int channel, char dimm_name, u16 val)
    748 {
    749 	if (INTEL_GEN(dev_priv) >= 10) {
    750 		dimm->size = cnl_get_dimm_size(val);
    751 		dimm->width = cnl_get_dimm_width(val);
    752 		dimm->ranks = cnl_get_dimm_ranks(val);
    753 	} else {
    754 		dimm->size = skl_get_dimm_size(val);
    755 		dimm->width = skl_get_dimm_width(val);
    756 		dimm->ranks = skl_get_dimm_ranks(val);
    757 	}
    758 
    759 	DRM_DEBUG_KMS("CH%u DIMM %c size: %u GB, width: X%u, ranks: %u, 16Gb DIMMs: %s\n",
    760 		      channel, dimm_name, dimm->size, dimm->width, dimm->ranks,
    761 		      yesno(skl_is_16gb_dimm(dimm)));
    762 }
    763 
    764 static int
    765 skl_dram_get_channel_info(struct drm_i915_private *dev_priv,
    766 			  struct dram_channel_info *ch,
    767 			  int channel, u32 val)
    768 {
    769 	skl_dram_get_dimm_info(dev_priv, &ch->dimm_l,
    770 			       channel, 'L', val & 0xffff);
    771 	skl_dram_get_dimm_info(dev_priv, &ch->dimm_s,
    772 			       channel, 'S', val >> 16);
    773 
    774 	if (ch->dimm_l.size == 0 && ch->dimm_s.size == 0) {
    775 		DRM_DEBUG_KMS("CH%u not populated\n", channel);
    776 		return -EINVAL;
    777 	}
    778 
    779 	if (ch->dimm_l.ranks == 2 || ch->dimm_s.ranks == 2)
    780 		ch->ranks = 2;
    781 	else if (ch->dimm_l.ranks == 1 && ch->dimm_s.ranks == 1)
    782 		ch->ranks = 2;
    783 	else
    784 		ch->ranks = 1;
    785 
    786 	ch->is_16gb_dimm =
    787 		skl_is_16gb_dimm(&ch->dimm_l) ||
    788 		skl_is_16gb_dimm(&ch->dimm_s);
    789 
    790 	DRM_DEBUG_KMS("CH%u ranks: %u, 16Gb DIMMs: %s\n",
    791 		      channel, ch->ranks, yesno(ch->is_16gb_dimm));
    792 
    793 	return 0;
    794 }
    795 
    796 static bool
    797 intel_is_dram_symmetric(const struct dram_channel_info *ch0,
    798 			const struct dram_channel_info *ch1)
    799 {
    800 	return !memcmp(ch0, ch1, sizeof(*ch0)) &&
    801 		(ch0->dimm_s.size == 0 ||
    802 		 !memcmp(&ch0->dimm_l, &ch0->dimm_s, sizeof(ch0->dimm_l)));
    803 }
    804 
    805 static int
    806 skl_dram_get_channels_info(struct drm_i915_private *dev_priv)
    807 {
    808 	struct dram_info *dram_info = &dev_priv->dram_info;
    809 	struct dram_channel_info ch0 = {}, ch1 = {};
    810 	u32 val;
    811 	int ret;
    812 
    813 	val = I915_READ(SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN);
    814 	ret = skl_dram_get_channel_info(dev_priv, &ch0, 0, val);
    815 	if (ret == 0)
    816 		dram_info->num_channels++;
    817 
    818 	val = I915_READ(SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN);
    819 	ret = skl_dram_get_channel_info(dev_priv, &ch1, 1, val);
    820 	if (ret == 0)
    821 		dram_info->num_channels++;
    822 
    823 	if (dram_info->num_channels == 0) {
    824 		DRM_INFO("Number of memory channels is zero\n");
    825 		return -EINVAL;
    826 	}
    827 
    828 	/*
    829 	 * If any of the channel is single rank channel, worst case output
    830 	 * will be same as if single rank memory, so consider single rank
    831 	 * memory.
    832 	 */
    833 	if (ch0.ranks == 1 || ch1.ranks == 1)
    834 		dram_info->ranks = 1;
    835 	else
    836 		dram_info->ranks = max(ch0.ranks, ch1.ranks);
    837 
    838 	if (dram_info->ranks == 0) {
    839 		DRM_INFO("couldn't get memory rank information\n");
    840 		return -EINVAL;
    841 	}
    842 
    843 	dram_info->is_16gb_dimm = ch0.is_16gb_dimm || ch1.is_16gb_dimm;
    844 
    845 	dram_info->symmetric_memory = intel_is_dram_symmetric(&ch0, &ch1);
    846 
    847 	DRM_DEBUG_KMS("Memory configuration is symmetric? %s\n",
    848 		      yesno(dram_info->symmetric_memory));
    849 	return 0;
    850 }
    851 
    852 static enum intel_dram_type
    853 skl_get_dram_type(struct drm_i915_private *dev_priv)
    854 {
    855 	u32 val;
    856 
    857 	val = I915_READ(SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN);
    858 
    859 	switch (val & SKL_DRAM_DDR_TYPE_MASK) {
    860 	case SKL_DRAM_DDR_TYPE_DDR3:
    861 		return INTEL_DRAM_DDR3;
    862 	case SKL_DRAM_DDR_TYPE_DDR4:
    863 		return INTEL_DRAM_DDR4;
    864 	case SKL_DRAM_DDR_TYPE_LPDDR3:
    865 		return INTEL_DRAM_LPDDR3;
    866 	case SKL_DRAM_DDR_TYPE_LPDDR4:
    867 		return INTEL_DRAM_LPDDR4;
    868 	default:
    869 		MISSING_CASE(val);
    870 		return INTEL_DRAM_UNKNOWN;
    871 	}
    872 }
    873 
    874 static int
    875 skl_get_dram_info(struct drm_i915_private *dev_priv)
    876 {
    877 	struct dram_info *dram_info = &dev_priv->dram_info;
    878 	u32 mem_freq_khz, val;
    879 	int ret;
    880 
    881 	dram_info->type = skl_get_dram_type(dev_priv);
    882 	DRM_DEBUG_KMS("DRAM type: %s\n", intel_dram_type_str(dram_info->type));
    883 
    884 	ret = skl_dram_get_channels_info(dev_priv);
    885 	if (ret)
    886 		return ret;
    887 
    888 	val = I915_READ(SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
    889 	mem_freq_khz = DIV_ROUND_UP((val & SKL_REQ_DATA_MASK) *
    890 				    SKL_MEMORY_FREQ_MULTIPLIER_HZ, 1000);
    891 
    892 	dram_info->bandwidth_kbps = dram_info->num_channels *
    893 							mem_freq_khz * 8;
    894 
    895 	if (dram_info->bandwidth_kbps == 0) {
    896 		DRM_INFO("Couldn't get system memory bandwidth\n");
    897 		return -EINVAL;
    898 	}
    899 
    900 	dram_info->valid = true;
    901 	return 0;
    902 }
    903 
    904 /* Returns Gb per DRAM device */
    905 static int bxt_get_dimm_size(u32 val)
    906 {
    907 	switch (val & BXT_DRAM_SIZE_MASK) {
    908 	case BXT_DRAM_SIZE_4GBIT:
    909 		return 4;
    910 	case BXT_DRAM_SIZE_6GBIT:
    911 		return 6;
    912 	case BXT_DRAM_SIZE_8GBIT:
    913 		return 8;
    914 	case BXT_DRAM_SIZE_12GBIT:
    915 		return 12;
    916 	case BXT_DRAM_SIZE_16GBIT:
    917 		return 16;
    918 	default:
    919 		MISSING_CASE(val);
    920 		return 0;
    921 	}
    922 }
    923 
    924 static int bxt_get_dimm_width(u32 val)
    925 {
    926 	if (!bxt_get_dimm_size(val))
    927 		return 0;
    928 
    929 	val = (val & BXT_DRAM_WIDTH_MASK) >> BXT_DRAM_WIDTH_SHIFT;
    930 
    931 	return 8 << val;
    932 }
    933 
    934 static int bxt_get_dimm_ranks(u32 val)
    935 {
    936 	if (!bxt_get_dimm_size(val))
    937 		return 0;
    938 
    939 	switch (val & BXT_DRAM_RANK_MASK) {
    940 	case BXT_DRAM_RANK_SINGLE:
    941 		return 1;
    942 	case BXT_DRAM_RANK_DUAL:
    943 		return 2;
    944 	default:
    945 		MISSING_CASE(val);
    946 		return 0;
    947 	}
    948 }
    949 
    950 static enum intel_dram_type bxt_get_dimm_type(u32 val)
    951 {
    952 	if (!bxt_get_dimm_size(val))
    953 		return INTEL_DRAM_UNKNOWN;
    954 
    955 	switch (val & BXT_DRAM_TYPE_MASK) {
    956 	case BXT_DRAM_TYPE_DDR3:
    957 		return INTEL_DRAM_DDR3;
    958 	case BXT_DRAM_TYPE_LPDDR3:
    959 		return INTEL_DRAM_LPDDR3;
    960 	case BXT_DRAM_TYPE_DDR4:
    961 		return INTEL_DRAM_DDR4;
    962 	case BXT_DRAM_TYPE_LPDDR4:
    963 		return INTEL_DRAM_LPDDR4;
    964 	default:
    965 		MISSING_CASE(val);
    966 		return INTEL_DRAM_UNKNOWN;
    967 	}
    968 }
    969 
    970 static void bxt_get_dimm_info(struct dram_dimm_info *dimm,
    971 			      u32 val)
    972 {
    973 	dimm->width = bxt_get_dimm_width(val);
    974 	dimm->ranks = bxt_get_dimm_ranks(val);
    975 
    976 	/*
    977 	 * Size in register is Gb per DRAM device. Convert to total
    978 	 * GB to match the way we report this for non-LP platforms.
    979 	 */
    980 	dimm->size = bxt_get_dimm_size(val) * intel_dimm_num_devices(dimm) / 8;
    981 }
    982 
    983 static int
    984 bxt_get_dram_info(struct drm_i915_private *dev_priv)
    985 {
    986 	struct dram_info *dram_info = &dev_priv->dram_info;
    987 	u32 dram_channels;
    988 	u32 mem_freq_khz, val;
    989 	u8 num_active_channels;
    990 	int i;
    991 
    992 	val = I915_READ(BXT_P_CR_MC_BIOS_REQ_0_0_0);
    993 	mem_freq_khz = DIV_ROUND_UP((val & BXT_REQ_DATA_MASK) *
    994 				    BXT_MEMORY_FREQ_MULTIPLIER_HZ, 1000);
    995 
    996 	dram_channels = val & BXT_DRAM_CHANNEL_ACTIVE_MASK;
    997 	num_active_channels = hweight32(dram_channels);
    998 
    999 	/* Each active bit represents 4-byte channel */
   1000 	dram_info->bandwidth_kbps = (mem_freq_khz * num_active_channels * 4);
   1001 
   1002 	if (dram_info->bandwidth_kbps == 0) {
   1003 		DRM_INFO("Couldn't get system memory bandwidth\n");
   1004 		return -EINVAL;
   1005 	}
   1006 
   1007 	/*
   1008 	 * Now read each DUNIT8/9/10/11 to check the rank of each dimms.
   1009 	 */
   1010 	for (i = BXT_D_CR_DRP0_DUNIT_START; i <= BXT_D_CR_DRP0_DUNIT_END; i++) {
   1011 		struct dram_dimm_info dimm;
   1012 		enum intel_dram_type type;
   1013 
   1014 		val = I915_READ(BXT_D_CR_DRP0_DUNIT(i));
   1015 		if (val == 0xFFFFFFFF)
   1016 			continue;
   1017 
   1018 		dram_info->num_channels++;
   1019 
   1020 		bxt_get_dimm_info(&dimm, val);
   1021 		type = bxt_get_dimm_type(val);
   1022 
   1023 		WARN_ON(type != INTEL_DRAM_UNKNOWN &&
   1024 			dram_info->type != INTEL_DRAM_UNKNOWN &&
   1025 			dram_info->type != type);
   1026 
   1027 		DRM_DEBUG_KMS("CH%u DIMM size: %u GB, width: X%u, ranks: %u, type: %s\n",
   1028 			      i - BXT_D_CR_DRP0_DUNIT_START,
   1029 			      dimm.size, dimm.width, dimm.ranks,
   1030 			      intel_dram_type_str(type));
   1031 
   1032 		/*
   1033 		 * If any of the channel is single rank channel,
   1034 		 * worst case output will be same as if single rank
   1035 		 * memory, so consider single rank memory.
   1036 		 */
   1037 		if (dram_info->ranks == 0)
   1038 			dram_info->ranks = dimm.ranks;
   1039 		else if (dimm.ranks == 1)
   1040 			dram_info->ranks = 1;
   1041 
   1042 		if (type != INTEL_DRAM_UNKNOWN)
   1043 			dram_info->type = type;
   1044 	}
   1045 
   1046 	if (dram_info->type == INTEL_DRAM_UNKNOWN ||
   1047 	    dram_info->ranks == 0) {
   1048 		DRM_INFO("couldn't get memory information\n");
   1049 		return -EINVAL;
   1050 	}
   1051 
   1052 	dram_info->valid = true;
   1053 	return 0;
   1054 }
   1055 
   1056 static void
   1057 intel_get_dram_info(struct drm_i915_private *dev_priv)
   1058 {
   1059 	struct dram_info *dram_info = &dev_priv->dram_info;
   1060 	int ret;
   1061 
   1062 	/*
   1063 	 * Assume 16Gb DIMMs are present until proven otherwise.
   1064 	 * This is only used for the level 0 watermark latency
   1065 	 * w/a which does not apply to bxt/glk.
   1066 	 */
   1067 	dram_info->is_16gb_dimm = !IS_GEN9_LP(dev_priv);
   1068 
   1069 	if (INTEL_GEN(dev_priv) < 9 || !HAS_DISPLAY(dev_priv))
   1070 		return;
   1071 
   1072 	if (IS_GEN9_LP(dev_priv))
   1073 		ret = bxt_get_dram_info(dev_priv);
   1074 	else
   1075 		ret = skl_get_dram_info(dev_priv);
   1076 	if (ret)
   1077 		return;
   1078 
   1079 	DRM_DEBUG_KMS("DRAM bandwidth: %u kBps, channels: %u\n",
   1080 		      dram_info->bandwidth_kbps,
   1081 		      dram_info->num_channels);
   1082 
   1083 	DRM_DEBUG_KMS("DRAM ranks: %u, 16Gb DIMMs: %s\n",
   1084 		      dram_info->ranks, yesno(dram_info->is_16gb_dimm));
   1085 }
   1086 
   1087 static u32 gen9_edram_size_mb(struct drm_i915_private *dev_priv, u32 cap)
   1088 {
   1089 	static const u8 ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 };
   1090 	static const u8 sets[4] = { 1, 1, 2, 2 };
   1091 
   1092 	return EDRAM_NUM_BANKS(cap) *
   1093 		ways[EDRAM_WAYS_IDX(cap)] *
   1094 		sets[EDRAM_SETS_IDX(cap)];
   1095 }
   1096 
   1097 static void edram_detect(struct drm_i915_private *dev_priv)
   1098 {
   1099 	u32 edram_cap = 0;
   1100 
   1101 	if (!(IS_HASWELL(dev_priv) ||
   1102 	      IS_BROADWELL(dev_priv) ||
   1103 	      INTEL_GEN(dev_priv) >= 9))
   1104 		return;
   1105 
   1106 	edram_cap = __raw_uncore_read32(&dev_priv->uncore, HSW_EDRAM_CAP);
   1107 
   1108 	/* NB: We can't write IDICR yet because we don't have gt funcs set up */
   1109 
   1110 	if (!(edram_cap & EDRAM_ENABLED))
   1111 		return;
   1112 
   1113 	/*
   1114 	 * The needed capability bits for size calculation are not there with
   1115 	 * pre gen9 so return 128MB always.
   1116 	 */
   1117 	if (INTEL_GEN(dev_priv) < 9)
   1118 		dev_priv->edram_size_mb = 128;
   1119 	else
   1120 		dev_priv->edram_size_mb =
   1121 			gen9_edram_size_mb(dev_priv, edram_cap);
   1122 
   1123 	dev_info(dev_priv->drm.dev,
   1124 		 "Found %uMB of eDRAM\n", dev_priv->edram_size_mb);
   1125 }
   1126 
   1127 /**
   1128  * i915_driver_hw_probe - setup state requiring device access
   1129  * @dev_priv: device private
   1130  *
   1131  * Setup state that requires accessing the device, but doesn't require
   1132  * exposing the driver via kernel internal or userspace interfaces.
   1133  */
   1134 static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
   1135 {
   1136 	struct pci_dev *pdev = dev_priv->drm.pdev;
   1137 	int ret;
   1138 
   1139 	if (i915_inject_probe_failure(dev_priv))
   1140 		return -ENODEV;
   1141 
   1142 	intel_device_info_runtime_init(dev_priv);
   1143 
   1144 	if (HAS_PPGTT(dev_priv)) {
   1145 		if (intel_vgpu_active(dev_priv) &&
   1146 		    !intel_vgpu_has_full_ppgtt(dev_priv)) {
   1147 			i915_report_error(dev_priv,
   1148 					  "incompatible vGPU found, support for isolated ppGTT required\n");
   1149 			return -ENXIO;
   1150 		}
   1151 	}
   1152 
   1153 	if (HAS_EXECLISTS(dev_priv)) {
   1154 		/*
   1155 		 * Older GVT emulation depends upon intercepting CSB mmio,
   1156 		 * which we no longer use, preferring to use the HWSP cache
   1157 		 * instead.
   1158 		 */
   1159 		if (intel_vgpu_active(dev_priv) &&
   1160 		    !intel_vgpu_has_hwsp_emulation(dev_priv)) {
   1161 			i915_report_error(dev_priv,
   1162 					  "old vGPU host found, support for HWSP emulation required\n");
   1163 			return -ENXIO;
   1164 		}
   1165 	}
   1166 
   1167 	intel_sanitize_options(dev_priv);
   1168 
   1169 	/* needs to be done before ggtt probe */
   1170 	edram_detect(dev_priv);
   1171 
   1172 	i915_perf_init(dev_priv);
   1173 
   1174 	ret = i915_ggtt_probe_hw(dev_priv);
   1175 	if (ret)
   1176 		goto err_perf;
   1177 
   1178 	ret = drm_fb_helper_remove_conflicting_pci_framebuffers(pdev, "inteldrmfb");
   1179 	if (ret)
   1180 		goto err_ggtt;
   1181 
   1182 	ret = i915_ggtt_init_hw(dev_priv);
   1183 	if (ret)
   1184 		goto err_ggtt;
   1185 
   1186 	ret = intel_memory_regions_hw_probe(dev_priv);
   1187 	if (ret)
   1188 		goto err_ggtt;
   1189 
   1190 	intel_gt_init_hw_early(&dev_priv->gt, &dev_priv->ggtt);
   1191 
   1192 	ret = i915_ggtt_enable_hw(dev_priv);
   1193 	if (ret) {
   1194 		DRM_ERROR("failed to enable GGTT\n");
   1195 		goto err_mem_regions;
   1196 	}
   1197 
   1198 	pci_set_master(pdev);
   1199 
   1200 	/*
   1201 	 * We don't have a max segment size, so set it to the max so sg's
   1202 	 * debugging layer doesn't complain
   1203 	 */
   1204 	dma_set_max_seg_size(&pdev->dev, UINT_MAX);
   1205 
   1206 	/* overlay on gen2 is broken and can't address above 1G */
   1207 	if (IS_GEN(dev_priv, 2)) {
   1208 		ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(30));
   1209 		if (ret) {
   1210 			DRM_ERROR("failed to set DMA mask\n");
   1211 
   1212 			goto err_mem_regions;
   1213 		}
   1214 	}
   1215 
   1216 	/* 965GM sometimes incorrectly writes to hardware status page (HWS)
   1217 	 * using 32bit addressing, overwriting memory if HWS is located
   1218 	 * above 4GB.
   1219 	 *
   1220 	 * The documentation also mentions an issue with undefined
   1221 	 * behaviour if any general state is accessed within a page above 4GB,
   1222 	 * which also needs to be handled carefully.
   1223 	 */
   1224 	if (IS_I965G(dev_priv) || IS_I965GM(dev_priv)) {
   1225 		ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
   1226 
   1227 		if (ret) {
   1228 			DRM_ERROR("failed to set DMA mask\n");
   1229 
   1230 			goto err_mem_regions;
   1231 		}
   1232 	}
   1233 
   1234 	pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY,
   1235 			   PM_QOS_DEFAULT_VALUE);
   1236 
   1237 	intel_gt_init_workarounds(dev_priv);
   1238 
   1239 	/* On the 945G/GM, the chipset reports the MSI capability on the
   1240 	 * integrated graphics even though the support isn't actually there
   1241 	 * according to the published specs.  It doesn't appear to function
   1242 	 * correctly in testing on 945G.
   1243 	 * This may be a side effect of MSI having been made available for PEG
   1244 	 * and the registers being closely associated.
   1245 	 *
   1246 	 * According to chipset errata, on the 965GM, MSI interrupts may
   1247 	 * be lost or delayed, and was defeatured. MSI interrupts seem to
   1248 	 * get lost on g4x as well, and interrupt delivery seems to stay
   1249 	 * properly dead afterwards. So we'll just disable them for all
   1250 	 * pre-gen5 chipsets.
   1251 	 *
   1252 	 * dp aux and gmbus irq on gen4 seems to be able to generate legacy
   1253 	 * interrupts even when in MSI mode. This results in spurious
   1254 	 * interrupt warnings if the legacy irq no. is shared with another
   1255 	 * device. The kernel then disables that interrupt source and so
   1256 	 * prevents the other device from working properly.
   1257 	 */
   1258 	if (INTEL_GEN(dev_priv) >= 5) {
   1259 		if (pci_enable_msi(pdev) < 0)
   1260 			DRM_DEBUG_DRIVER("can't enable MSI");
   1261 	}
   1262 
   1263 	ret = intel_gvt_init(dev_priv);
   1264 	if (ret)
   1265 		goto err_msi;
   1266 
   1267 	intel_opregion_setup(dev_priv);
   1268 	/*
   1269 	 * Fill the dram structure to get the system raw bandwidth and
   1270 	 * dram info. This will be used for memory latency calculation.
   1271 	 */
   1272 	intel_get_dram_info(dev_priv);
   1273 
   1274 	intel_bw_init_hw(dev_priv);
   1275 
   1276 	return 0;
   1277 
   1278 err_msi:
   1279 	if (pdev->msi_enabled)
   1280 		pci_disable_msi(pdev);
   1281 	pm_qos_remove_request(&dev_priv->pm_qos);
   1282 err_mem_regions:
   1283 	intel_memory_regions_driver_release(dev_priv);
   1284 err_ggtt:
   1285 	i915_ggtt_driver_release(dev_priv);
   1286 err_perf:
   1287 	i915_perf_fini(dev_priv);
   1288 	return ret;
   1289 }
   1290 
   1291 /**
   1292  * i915_driver_hw_remove - cleanup the setup done in i915_driver_hw_probe()
   1293  * @dev_priv: device private
   1294  */
   1295 static void i915_driver_hw_remove(struct drm_i915_private *dev_priv)
   1296 {
   1297 	struct pci_dev *pdev = dev_priv->drm.pdev;
   1298 
   1299 	i915_perf_fini(dev_priv);
   1300 
   1301 	if (pdev->msi_enabled)
   1302 		pci_disable_msi(pdev);
   1303 
   1304 	pm_qos_remove_request(&dev_priv->pm_qos);
   1305 }
   1306 
   1307 /**
   1308  * i915_driver_register - register the driver with the rest of the system
   1309  * @dev_priv: device private
   1310  *
   1311  * Perform any steps necessary to make the driver available via kernel
   1312  * internal or userspace interfaces.
   1313  */
   1314 static void i915_driver_register(struct drm_i915_private *dev_priv)
   1315 {
   1316 	struct drm_device *dev = &dev_priv->drm;
   1317 
   1318 	i915_gem_driver_register(dev_priv);
   1319 	i915_pmu_register(dev_priv);
   1320 
   1321 	/*
   1322 	 * Notify a valid surface after modesetting,
   1323 	 * when running inside a VM.
   1324 	 */
   1325 	if (intel_vgpu_active(dev_priv))
   1326 		I915_WRITE(vgtif_reg(display_ready), VGT_DRV_DISPLAY_READY);
   1327 
   1328 	/* Reveal our presence to userspace */
   1329 	if (drm_dev_register(dev, 0) == 0) {
   1330 		i915_debugfs_register(dev_priv);
   1331 		i915_setup_sysfs(dev_priv);
   1332 
   1333 		/* Depends on sysfs having been initialized */
   1334 		i915_perf_register(dev_priv);
   1335 	} else
   1336 		DRM_ERROR("Failed to register driver for userspace access!\n");
   1337 
   1338 	if (HAS_DISPLAY(dev_priv) && INTEL_DISPLAY_ENABLED(dev_priv)) {
   1339 		/* Must be done after probing outputs */
   1340 		intel_opregion_register(dev_priv);
   1341 		acpi_video_register();
   1342 	}
   1343 
   1344 	intel_gt_driver_register(&dev_priv->gt);
   1345 
   1346 	intel_audio_init(dev_priv);
   1347 
   1348 	/*
   1349 	 * Some ports require correctly set-up hpd registers for detection to
   1350 	 * work properly (leading to ghost connected connector status), e.g. VGA
   1351 	 * on gm45.  Hence we can only set up the initial fbdev config after hpd
   1352 	 * irqs are fully enabled. We do it last so that the async config
   1353 	 * cannot run before the connectors are registered.
   1354 	 */
   1355 	intel_fbdev_initial_config_async(dev);
   1356 
   1357 	/*
   1358 	 * We need to coordinate the hotplugs with the asynchronous fbdev
   1359 	 * configuration, for which we use the fbdev->async_cookie.
   1360 	 */
   1361 	if (HAS_DISPLAY(dev_priv) && INTEL_DISPLAY_ENABLED(dev_priv))
   1362 		drm_kms_helper_poll_init(dev);
   1363 
   1364 	intel_power_domains_enable(dev_priv);
   1365 	intel_runtime_pm_enable(&dev_priv->runtime_pm);
   1366 }
   1367 
   1368 /**
   1369  * i915_driver_unregister - cleanup the registration done in i915_driver_regiser()
   1370  * @dev_priv: device private
   1371  */
   1372 static void i915_driver_unregister(struct drm_i915_private *dev_priv)
   1373 {
   1374 	intel_runtime_pm_disable(&dev_priv->runtime_pm);
   1375 	intel_power_domains_disable(dev_priv);
   1376 
   1377 	intel_fbdev_unregister(dev_priv);
   1378 	intel_audio_deinit(dev_priv);
   1379 
   1380 	/*
   1381 	 * After flushing the fbdev (incl. a late async config which will
   1382 	 * have delayed queuing of a hotplug event), then flush the hotplug
   1383 	 * events.
   1384 	 */
   1385 	drm_kms_helper_poll_fini(&dev_priv->drm);
   1386 
   1387 	intel_gt_driver_unregister(&dev_priv->gt);
   1388 	acpi_video_unregister();
   1389 	intel_opregion_unregister(dev_priv);
   1390 
   1391 	i915_perf_unregister(dev_priv);
   1392 	i915_pmu_unregister(dev_priv);
   1393 
   1394 	i915_teardown_sysfs(dev_priv);
   1395 	drm_dev_unplug(&dev_priv->drm);
   1396 
   1397 	i915_gem_driver_unregister(dev_priv);
   1398 }
   1399 
   1400 static void i915_welcome_messages(struct drm_i915_private *dev_priv)
   1401 {
   1402 	if (drm_debug_enabled(DRM_UT_DRIVER)) {
   1403 		struct drm_printer p = drm_debug_printer("i915 device info:");
   1404 
   1405 		drm_printf(&p, "pciid=0x%04x rev=0x%02x platform=%s (subplatform=0x%x) gen=%i\n",
   1406 			   INTEL_DEVID(dev_priv),
   1407 			   INTEL_REVID(dev_priv),
   1408 			   intel_platform_name(INTEL_INFO(dev_priv)->platform),
   1409 			   intel_subplatform(RUNTIME_INFO(dev_priv),
   1410 					     INTEL_INFO(dev_priv)->platform),
   1411 			   INTEL_GEN(dev_priv));
   1412 
   1413 		intel_device_info_print_static(INTEL_INFO(dev_priv), &p);
   1414 		intel_device_info_print_runtime(RUNTIME_INFO(dev_priv), &p);
   1415 	}
   1416 
   1417 	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG))
   1418 		DRM_INFO("DRM_I915_DEBUG enabled\n");
   1419 	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
   1420 		DRM_INFO("DRM_I915_DEBUG_GEM enabled\n");
   1421 	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM))
   1422 		DRM_INFO("DRM_I915_DEBUG_RUNTIME_PM enabled\n");
   1423 }
   1424 
   1425 static struct drm_i915_private *
   1426 i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent)
   1427 {
   1428 	const struct intel_device_info *match_info =
   1429 		(struct intel_device_info *)ent->driver_data;
   1430 	struct intel_device_info *device_info;
   1431 	struct drm_i915_private *i915;
   1432 	int err;
   1433 
   1434 	i915 = kzalloc(sizeof(*i915), GFP_KERNEL);
   1435 	if (!i915)
   1436 		return ERR_PTR(-ENOMEM);
   1437 
   1438 	err = drm_dev_init(&i915->drm, &driver, &pdev->dev);
   1439 	if (err) {
   1440 		kfree(i915);
   1441 		return ERR_PTR(err);
   1442 	}
   1443 
   1444 	i915->drm.dev_private = i915;
   1445 
   1446 	i915->drm.pdev = pdev;
   1447 	pci_set_drvdata(pdev, i915);
   1448 
   1449 	/* Setup the write-once "constant" device info */
   1450 	device_info = mkwrite_device_info(i915);
   1451 	memcpy(device_info, match_info, sizeof(*device_info));
   1452 	RUNTIME_INFO(i915)->device_id = pdev->device;
   1453 
   1454 	BUG_ON(device_info->gen > BITS_PER_TYPE(device_info->gen_mask));
   1455 
   1456 	return i915;
   1457 }
   1458 
   1459 static void i915_driver_destroy(struct drm_i915_private *i915)
   1460 {
   1461 	struct pci_dev *pdev = i915->drm.pdev;
   1462 
   1463 	drm_dev_fini(&i915->drm);
   1464 	kfree(i915);
   1465 
   1466 	/* And make sure we never chase our dangling pointer from pci_dev */
   1467 	pci_set_drvdata(pdev, NULL);
   1468 }
   1469 
   1470 /**
   1471  * i915_driver_probe - setup chip and create an initial config
   1472  * @pdev: PCI device
   1473  * @ent: matching PCI ID entry
   1474  *
   1475  * The driver probe routine has to do several things:
   1476  *   - drive output discovery via intel_modeset_init()
   1477  *   - initialize the memory manager
   1478  *   - allocate initial config memory
   1479  *   - setup the DRM framebuffer with the allocated memory
   1480  */
   1481 int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
   1482 {
   1483 	const struct intel_device_info *match_info =
   1484 		(struct intel_device_info *)ent->driver_data;
   1485 	struct drm_i915_private *dev_priv;
   1486 	int ret;
   1487 
   1488 	dev_priv = i915_driver_create(pdev, ent);
   1489 	if (IS_ERR(dev_priv))
   1490 		return PTR_ERR(dev_priv);
   1491 
   1492 	/* Disable nuclear pageflip by default on pre-ILK */
   1493 	if (!i915_modparams.nuclear_pageflip && match_info->gen < 5)
   1494 		dev_priv->drm.driver_features &= ~DRIVER_ATOMIC;
   1495 
   1496 	/*
   1497 	 * Check if we support fake LMEM -- for now we only unleash this for
   1498 	 * the live selftests(test-and-exit).
   1499 	 */
   1500 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
   1501 	if (IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM)) {
   1502 		if (INTEL_GEN(dev_priv) >= 9 && i915_selftest.live < 0 &&
   1503 		    i915_modparams.fake_lmem_start) {
   1504 			mkwrite_device_info(dev_priv)->memory_regions =
   1505 				REGION_SMEM | REGION_LMEM | REGION_STOLEN;
   1506 			mkwrite_device_info(dev_priv)->is_dgfx = true;
   1507 			GEM_BUG_ON(!HAS_LMEM(dev_priv));
   1508 			GEM_BUG_ON(!IS_DGFX(dev_priv));
   1509 		}
   1510 	}
   1511 #endif
   1512 
   1513 	ret = pci_enable_device(pdev);
   1514 	if (ret)
   1515 		goto out_fini;
   1516 
   1517 	ret = i915_driver_early_probe(dev_priv);
   1518 	if (ret < 0)
   1519 		goto out_pci_disable;
   1520 
   1521 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
   1522 
   1523 	i915_detect_vgpu(dev_priv);
   1524 
   1525 	ret = i915_driver_mmio_probe(dev_priv);
   1526 	if (ret < 0)
   1527 		goto out_runtime_pm_put;
   1528 
   1529 	ret = i915_driver_hw_probe(dev_priv);
   1530 	if (ret < 0)
   1531 		goto out_cleanup_mmio;
   1532 
   1533 	ret = i915_driver_modeset_probe(dev_priv);
   1534 	if (ret < 0)
   1535 		goto out_cleanup_hw;
   1536 
   1537 	i915_driver_register(dev_priv);
   1538 
   1539 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
   1540 
   1541 	i915_welcome_messages(dev_priv);
   1542 
   1543 	return 0;
   1544 
   1545 out_cleanup_hw:
   1546 	i915_driver_hw_remove(dev_priv);
   1547 	intel_memory_regions_driver_release(dev_priv);
   1548 	i915_ggtt_driver_release(dev_priv);
   1549 out_cleanup_mmio:
   1550 	i915_driver_mmio_release(dev_priv);
   1551 out_runtime_pm_put:
   1552 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
   1553 	i915_driver_late_release(dev_priv);
   1554 out_pci_disable:
   1555 	pci_disable_device(pdev);
   1556 out_fini:
   1557 	i915_probe_error(dev_priv, "Device initialization failed (%d)\n", ret);
   1558 	i915_driver_destroy(dev_priv);
   1559 	return ret;
   1560 }
   1561 
   1562 void i915_driver_remove(struct drm_i915_private *i915)
   1563 {
   1564 	disable_rpm_wakeref_asserts(&i915->runtime_pm);
   1565 
   1566 	i915_driver_unregister(i915);
   1567 
   1568 	/*
   1569 	 * After unregistering the device to prevent any new users, cancel
   1570 	 * all in-flight requests so that we can quickly unbind the active
   1571 	 * resources.
   1572 	 */
   1573 	intel_gt_set_wedged(&i915->gt);
   1574 
   1575 	/* Flush any external code that still may be under the RCU lock */
   1576 	synchronize_rcu();
   1577 
   1578 	i915_gem_suspend(i915);
   1579 
   1580 	drm_atomic_helper_shutdown(&i915->drm);
   1581 
   1582 	intel_gvt_driver_remove(i915);
   1583 
   1584 	i915_driver_modeset_remove(i915);
   1585 
   1586 	i915_reset_error_state(i915);
   1587 	i915_gem_driver_remove(i915);
   1588 
   1589 	intel_power_domains_driver_remove(i915);
   1590 
   1591 	i915_driver_hw_remove(i915);
   1592 
   1593 	enable_rpm_wakeref_asserts(&i915->runtime_pm);
   1594 }
   1595 
   1596 static void i915_driver_release(struct drm_device *dev)
   1597 {
   1598 	struct drm_i915_private *dev_priv = to_i915(dev);
   1599 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
   1600 
   1601 	disable_rpm_wakeref_asserts(rpm);
   1602 
   1603 	i915_gem_driver_release(dev_priv);
   1604 
   1605 	intel_memory_regions_driver_release(dev_priv);
   1606 	i915_ggtt_driver_release(dev_priv);
   1607 
   1608 	i915_driver_mmio_release(dev_priv);
   1609 
   1610 	enable_rpm_wakeref_asserts(rpm);
   1611 	intel_runtime_pm_driver_release(rpm);
   1612 
   1613 	i915_driver_late_release(dev_priv);
   1614 	i915_driver_destroy(dev_priv);
   1615 }
   1616 
   1617 static int i915_driver_open(struct drm_device *dev, struct drm_file *file)
   1618 {
   1619 	struct drm_i915_private *i915 = to_i915(dev);
   1620 	int ret;
   1621 
   1622 	ret = i915_gem_open(i915, file);
   1623 	if (ret)
   1624 		return ret;
   1625 
   1626 	return 0;
   1627 }
   1628 
   1629 /**
   1630  * i915_driver_lastclose - clean up after all DRM clients have exited
   1631  * @dev: DRM device
   1632  *
   1633  * Take care of cleaning up after all DRM clients have exited.  In the
   1634  * mode setting case, we want to restore the kernel's initial mode (just
   1635  * in case the last client left us in a bad state).
   1636  *
   1637  * Additionally, in the non-mode setting case, we'll tear down the GTT
   1638  * and DMA structures, since the kernel won't be using them, and clea
   1639  * up any GEM state.
   1640  */
   1641 static void i915_driver_lastclose(struct drm_device *dev)
   1642 {
   1643 	intel_fbdev_restore_mode(dev);
   1644 	vga_switcheroo_process_delayed_switch();
   1645 }
   1646 
   1647 static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
   1648 {
   1649 	struct drm_i915_file_private *file_priv = file->driver_priv;
   1650 
   1651 	i915_gem_context_close(file);
   1652 	i915_gem_release(dev, file);
   1653 
   1654 	kfree_rcu(file_priv, rcu);
   1655 
   1656 	/* Catch up with all the deferred frees from "this" client */
   1657 	i915_gem_flush_free_objects(to_i915(dev));
   1658 }
   1659 
   1660 static void intel_suspend_encoders(struct drm_i915_private *dev_priv)
   1661 {
   1662 	struct drm_device *dev = &dev_priv->drm;
   1663 	struct intel_encoder *encoder;
   1664 
   1665 	drm_modeset_lock_all(dev);
   1666 	for_each_intel_encoder(dev, encoder)
   1667 		if (encoder->suspend)
   1668 			encoder->suspend(encoder);
   1669 	drm_modeset_unlock_all(dev);
   1670 }
   1671 
   1672 static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
   1673 			      bool rpm_resume);
   1674 static int vlv_suspend_complete(struct drm_i915_private *dev_priv);
   1675 
   1676 static bool suspend_to_idle(struct drm_i915_private *dev_priv)
   1677 {
   1678 #if IS_ENABLED(CONFIG_ACPI_SLEEP)
   1679 	if (acpi_target_system_state() < ACPI_STATE_S3)
   1680 		return true;
   1681 #endif
   1682 	return false;
   1683 }
   1684 
   1685 static int i915_drm_prepare(struct drm_device *dev)
   1686 {
   1687 	struct drm_i915_private *i915 = to_i915(dev);
   1688 
   1689 	/*
   1690 	 * NB intel_display_suspend() may issue new requests after we've
   1691 	 * ostensibly marked the GPU as ready-to-sleep here. We need to
   1692 	 * split out that work and pull it forward so that after point,
   1693 	 * the GPU is not woken again.
   1694 	 */
   1695 	i915_gem_suspend(i915);
   1696 
   1697 	return 0;
   1698 }
   1699 
   1700 static int i915_drm_suspend(struct drm_device *dev)
   1701 {
   1702 	struct drm_i915_private *dev_priv = to_i915(dev);
   1703 	struct pci_dev *pdev = dev_priv->drm.pdev;
   1704 	pci_power_t opregion_target_state;
   1705 
   1706 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
   1707 
   1708 	/* We do a lot of poking in a lot of registers, make sure they work
   1709 	 * properly. */
   1710 	intel_power_domains_disable(dev_priv);
   1711 
   1712 	drm_kms_helper_poll_disable(dev);
   1713 
   1714 	pci_save_state(pdev);
   1715 
   1716 	intel_display_suspend(dev);
   1717 
   1718 	intel_dp_mst_suspend(dev_priv);
   1719 
   1720 	intel_runtime_pm_disable_interrupts(dev_priv);
   1721 	intel_hpd_cancel_work(dev_priv);
   1722 
   1723 	intel_suspend_encoders(dev_priv);
   1724 
   1725 	intel_suspend_hw(dev_priv);
   1726 
   1727 	i915_gem_suspend_gtt_mappings(dev_priv);
   1728 
   1729 	i915_save_state(dev_priv);
   1730 
   1731 	opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold;
   1732 	intel_opregion_suspend(dev_priv, opregion_target_state);
   1733 
   1734 	intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true);
   1735 
   1736 	dev_priv->suspend_count++;
   1737 
   1738 	intel_csr_ucode_suspend(dev_priv);
   1739 
   1740 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
   1741 
   1742 	return 0;
   1743 }
   1744 
   1745 static enum i915_drm_suspend_mode
   1746 get_suspend_mode(struct drm_i915_private *dev_priv, bool hibernate)
   1747 {
   1748 	if (hibernate)
   1749 		return I915_DRM_SUSPEND_HIBERNATE;
   1750 
   1751 	if (suspend_to_idle(dev_priv))
   1752 		return I915_DRM_SUSPEND_IDLE;
   1753 
   1754 	return I915_DRM_SUSPEND_MEM;
   1755 }
   1756 
   1757 static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
   1758 {
   1759 	struct drm_i915_private *dev_priv = to_i915(dev);
   1760 	struct pci_dev *pdev = dev_priv->drm.pdev;
   1761 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
   1762 	int ret = 0;
   1763 
   1764 	disable_rpm_wakeref_asserts(rpm);
   1765 
   1766 	i915_gem_suspend_late(dev_priv);
   1767 
   1768 	intel_uncore_suspend(&dev_priv->uncore);
   1769 
   1770 	intel_power_domains_suspend(dev_priv,
   1771 				    get_suspend_mode(dev_priv, hibernation));
   1772 
   1773 	intel_display_power_suspend_late(dev_priv);
   1774 
   1775 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
   1776 		ret = vlv_suspend_complete(dev_priv);
   1777 
   1778 	if (ret) {
   1779 		DRM_ERROR("Suspend complete failed: %d\n", ret);
   1780 		intel_power_domains_resume(dev_priv);
   1781 
   1782 		goto out;
   1783 	}
   1784 
   1785 	pci_disable_device(pdev);
   1786 	/*
   1787 	 * During hibernation on some platforms the BIOS may try to access
   1788 	 * the device even though it's already in D3 and hang the machine. So
   1789 	 * leave the device in D0 on those platforms and hope the BIOS will
   1790 	 * power down the device properly. The issue was seen on multiple old
   1791 	 * GENs with different BIOS vendors, so having an explicit blacklist
   1792 	 * is inpractical; apply the workaround on everything pre GEN6. The
   1793 	 * platforms where the issue was seen:
   1794 	 * Lenovo Thinkpad X301, X61s, X60, T60, X41
   1795 	 * Fujitsu FSC S7110
   1796 	 * Acer Aspire 1830T
   1797 	 */
   1798 	if (!(hibernation && INTEL_GEN(dev_priv) < 6))
   1799 		pci_set_power_state(pdev, PCI_D3hot);
   1800 
   1801 out:
   1802 	enable_rpm_wakeref_asserts(rpm);
   1803 	if (!dev_priv->uncore.user_forcewake_count)
   1804 		intel_runtime_pm_driver_release(rpm);
   1805 
   1806 	return ret;
   1807 }
   1808 
   1809 int i915_suspend_switcheroo(struct drm_i915_private *i915, pm_message_t state)
   1810 {
   1811 	int error;
   1812 
   1813 	if (WARN_ON_ONCE(state.event != PM_EVENT_SUSPEND &&
   1814 			 state.event != PM_EVENT_FREEZE))
   1815 		return -EINVAL;
   1816 
   1817 	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
   1818 		return 0;
   1819 
   1820 	error = i915_drm_suspend(&i915->drm);
   1821 	if (error)
   1822 		return error;
   1823 
   1824 	return i915_drm_suspend_late(&i915->drm, false);
   1825 }
   1826 
   1827 static int i915_drm_resume(struct drm_device *dev)
   1828 {
   1829 	struct drm_i915_private *dev_priv = to_i915(dev);
   1830 	int ret;
   1831 
   1832 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
   1833 
   1834 	sanitize_gpu(dev_priv);
   1835 
   1836 	ret = i915_ggtt_enable_hw(dev_priv);
   1837 	if (ret)
   1838 		DRM_ERROR("failed to re-enable GGTT\n");
   1839 
   1840 	i915_gem_restore_gtt_mappings(dev_priv);
   1841 	i915_gem_restore_fences(&dev_priv->ggtt);
   1842 
   1843 	intel_csr_ucode_resume(dev_priv);
   1844 
   1845 	i915_restore_state(dev_priv);
   1846 	intel_pps_unlock_regs_wa(dev_priv);
   1847 
   1848 	intel_init_pch_refclk(dev_priv);
   1849 
   1850 	/*
   1851 	 * Interrupts have to be enabled before any batches are run. If not the
   1852 	 * GPU will hang. i915_gem_init_hw() will initiate batches to
   1853 	 * update/restore the context.
   1854 	 *
   1855 	 * drm_mode_config_reset() needs AUX interrupts.
   1856 	 *
   1857 	 * Modeset enabling in intel_modeset_init_hw() also needs working
   1858 	 * interrupts.
   1859 	 */
   1860 	intel_runtime_pm_enable_interrupts(dev_priv);
   1861 
   1862 	drm_mode_config_reset(dev);
   1863 
   1864 	i915_gem_resume(dev_priv);
   1865 
   1866 	intel_modeset_init_hw(dev_priv);
   1867 	intel_init_clock_gating(dev_priv);
   1868 
   1869 	spin_lock_irq(&dev_priv->irq_lock);
   1870 	if (dev_priv->display.hpd_irq_setup)
   1871 		dev_priv->display.hpd_irq_setup(dev_priv);
   1872 	spin_unlock_irq(&dev_priv->irq_lock);
   1873 
   1874 	intel_dp_mst_resume(dev_priv);
   1875 
   1876 	intel_display_resume(dev);
   1877 
   1878 	drm_kms_helper_poll_enable(dev);
   1879 
   1880 	/*
   1881 	 * ... but also need to make sure that hotplug processing
   1882 	 * doesn't cause havoc. Like in the driver load code we don't
   1883 	 * bother with the tiny race here where we might lose hotplug
   1884 	 * notifications.
   1885 	 * */
   1886 	intel_hpd_init(dev_priv);
   1887 
   1888 	intel_opregion_resume(dev_priv);
   1889 
   1890 	intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false);
   1891 
   1892 	intel_power_domains_enable(dev_priv);
   1893 
   1894 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
   1895 
   1896 	return 0;
   1897 }
   1898 
   1899 static int i915_drm_resume_early(struct drm_device *dev)
   1900 {
   1901 	struct drm_i915_private *dev_priv = to_i915(dev);
   1902 	struct pci_dev *pdev = dev_priv->drm.pdev;
   1903 	int ret;
   1904 
   1905 	/*
   1906 	 * We have a resume ordering issue with the snd-hda driver also
   1907 	 * requiring our device to be power up. Due to the lack of a
   1908 	 * parent/child relationship we currently solve this with an early
   1909 	 * resume hook.
   1910 	 *
   1911 	 * FIXME: This should be solved with a special hdmi sink device or
   1912 	 * similar so that power domains can be employed.
   1913 	 */
   1914 
   1915 	/*
   1916 	 * Note that we need to set the power state explicitly, since we
   1917 	 * powered off the device during freeze and the PCI core won't power
   1918 	 * it back up for us during thaw. Powering off the device during
   1919 	 * freeze is not a hard requirement though, and during the
   1920 	 * suspend/resume phases the PCI core makes sure we get here with the
   1921 	 * device powered on. So in case we change our freeze logic and keep
   1922 	 * the device powered we can also remove the following set power state
   1923 	 * call.
   1924 	 */
   1925 	ret = pci_set_power_state(pdev, PCI_D0);
   1926 	if (ret) {
   1927 		DRM_ERROR("failed to set PCI D0 power state (%d)\n", ret);
   1928 		return ret;
   1929 	}
   1930 
   1931 	/*
   1932 	 * Note that pci_enable_device() first enables any parent bridge
   1933 	 * device and only then sets the power state for this device. The
   1934 	 * bridge enabling is a nop though, since bridge devices are resumed
   1935 	 * first. The order of enabling power and enabling the device is
   1936 	 * imposed by the PCI core as described above, so here we preserve the
   1937 	 * same order for the freeze/thaw phases.
   1938 	 *
   1939 	 * TODO: eventually we should remove pci_disable_device() /
   1940 	 * pci_enable_enable_device() from suspend/resume. Due to how they
   1941 	 * depend on the device enable refcount we can't anyway depend on them
   1942 	 * disabling/enabling the device.
   1943 	 */
   1944 	if (pci_enable_device(pdev))
   1945 		return -EIO;
   1946 
   1947 	pci_set_master(pdev);
   1948 
   1949 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
   1950 
   1951 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
   1952 		ret = vlv_resume_prepare(dev_priv, false);
   1953 	if (ret)
   1954 		DRM_ERROR("Resume prepare failed: %d, continuing anyway\n",
   1955 			  ret);
   1956 
   1957 	intel_uncore_resume_early(&dev_priv->uncore);
   1958 
   1959 	intel_gt_check_and_clear_faults(&dev_priv->gt);
   1960 
   1961 	intel_display_power_resume_early(dev_priv);
   1962 
   1963 	intel_power_domains_resume(dev_priv);
   1964 
   1965 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
   1966 
   1967 	return ret;
   1968 }
   1969 
   1970 int i915_resume_switcheroo(struct drm_i915_private *i915)
   1971 {
   1972 	int ret;
   1973 
   1974 	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
   1975 		return 0;
   1976 
   1977 	ret = i915_drm_resume_early(&i915->drm);
   1978 	if (ret)
   1979 		return ret;
   1980 
   1981 	return i915_drm_resume(&i915->drm);
   1982 }
   1983 
   1984 static int i915_pm_prepare(struct device *kdev)
   1985 {
   1986 	struct drm_i915_private *i915 = kdev_to_i915(kdev);
   1987 
   1988 	if (!i915) {
   1989 		dev_err(kdev, "DRM not initialized, aborting suspend.\n");
   1990 		return -ENODEV;
   1991 	}
   1992 
   1993 	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
   1994 		return 0;
   1995 
   1996 	return i915_drm_prepare(&i915->drm);
   1997 }
   1998 
   1999 static int i915_pm_suspend(struct device *kdev)
   2000 {
   2001 	struct drm_i915_private *i915 = kdev_to_i915(kdev);
   2002 
   2003 	if (!i915) {
   2004 		dev_err(kdev, "DRM not initialized, aborting suspend.\n");
   2005 		return -ENODEV;
   2006 	}
   2007 
   2008 	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
   2009 		return 0;
   2010 
   2011 	return i915_drm_suspend(&i915->drm);
   2012 }
   2013 
   2014 static int i915_pm_suspend_late(struct device *kdev)
   2015 {
   2016 	struct drm_i915_private *i915 = kdev_to_i915(kdev);
   2017 
   2018 	/*
   2019 	 * We have a suspend ordering issue with the snd-hda driver also
   2020 	 * requiring our device to be power up. Due to the lack of a
   2021 	 * parent/child relationship we currently solve this with an late
   2022 	 * suspend hook.
   2023 	 *
   2024 	 * FIXME: This should be solved with a special hdmi sink device or
   2025 	 * similar so that power domains can be employed.
   2026 	 */
   2027 	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
   2028 		return 0;
   2029 
   2030 	return i915_drm_suspend_late(&i915->drm, false);
   2031 }
   2032 
   2033 static int i915_pm_poweroff_late(struct device *kdev)
   2034 {
   2035 	struct drm_i915_private *i915 = kdev_to_i915(kdev);
   2036 
   2037 	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
   2038 		return 0;
   2039 
   2040 	return i915_drm_suspend_late(&i915->drm, true);
   2041 }
   2042 
   2043 static int i915_pm_resume_early(struct device *kdev)
   2044 {
   2045 	struct drm_i915_private *i915 = kdev_to_i915(kdev);
   2046 
   2047 	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
   2048 		return 0;
   2049 
   2050 	return i915_drm_resume_early(&i915->drm);
   2051 }
   2052 
   2053 static int i915_pm_resume(struct device *kdev)
   2054 {
   2055 	struct drm_i915_private *i915 = kdev_to_i915(kdev);
   2056 
   2057 	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
   2058 		return 0;
   2059 
   2060 	return i915_drm_resume(&i915->drm);
   2061 }
   2062 
   2063 /* freeze: before creating the hibernation_image */
   2064 static int i915_pm_freeze(struct device *kdev)
   2065 {
   2066 	struct drm_i915_private *i915 = kdev_to_i915(kdev);
   2067 	int ret;
   2068 
   2069 	if (i915->drm.switch_power_state != DRM_SWITCH_POWER_OFF) {
   2070 		ret = i915_drm_suspend(&i915->drm);
   2071 		if (ret)
   2072 			return ret;
   2073 	}
   2074 
   2075 	ret = i915_gem_freeze(i915);
   2076 	if (ret)
   2077 		return ret;
   2078 
   2079 	return 0;
   2080 }
   2081 
   2082 static int i915_pm_freeze_late(struct device *kdev)
   2083 {
   2084 	struct drm_i915_private *i915 = kdev_to_i915(kdev);
   2085 	int ret;
   2086 
   2087 	if (i915->drm.switch_power_state != DRM_SWITCH_POWER_OFF) {
   2088 		ret = i915_drm_suspend_late(&i915->drm, true);
   2089 		if (ret)
   2090 			return ret;
   2091 	}
   2092 
   2093 	ret = i915_gem_freeze_late(i915);
   2094 	if (ret)
   2095 		return ret;
   2096 
   2097 	return 0;
   2098 }
   2099 
   2100 /* thaw: called after creating the hibernation image, but before turning off. */
   2101 static int i915_pm_thaw_early(struct device *kdev)
   2102 {
   2103 	return i915_pm_resume_early(kdev);
   2104 }
   2105 
   2106 static int i915_pm_thaw(struct device *kdev)
   2107 {
   2108 	return i915_pm_resume(kdev);
   2109 }
   2110 
   2111 /* restore: called after loading the hibernation image. */
   2112 static int i915_pm_restore_early(struct device *kdev)
   2113 {
   2114 	return i915_pm_resume_early(kdev);
   2115 }
   2116 
   2117 static int i915_pm_restore(struct device *kdev)
   2118 {
   2119 	return i915_pm_resume(kdev);
   2120 }
   2121 
   2122 /*
   2123  * Save all Gunit registers that may be lost after a D3 and a subsequent
   2124  * S0i[R123] transition. The list of registers needing a save/restore is
   2125  * defined in the VLV2_S0IXRegs document. This documents marks all Gunit
   2126  * registers in the following way:
   2127  * - Driver: saved/restored by the driver
   2128  * - Punit : saved/restored by the Punit firmware
   2129  * - No, w/o marking: no need to save/restore, since the register is R/O or
   2130  *                    used internally by the HW in a way that doesn't depend
   2131  *                    keeping the content across a suspend/resume.
   2132  * - Debug : used for debugging
   2133  *
   2134  * We save/restore all registers marked with 'Driver', with the following
   2135  * exceptions:
   2136  * - Registers out of use, including also registers marked with 'Debug'.
   2137  *   These have no effect on the driver's operation, so we don't save/restore
   2138  *   them to reduce the overhead.
   2139  * - Registers that are fully setup by an initialization function called from
   2140  *   the resume path. For example many clock gating and RPS/RC6 registers.
   2141  * - Registers that provide the right functionality with their reset defaults.
   2142  *
   2143  * TODO: Except for registers that based on the above 3 criteria can be safely
   2144  * ignored, we save/restore all others, practically treating the HW context as
   2145  * a black-box for the driver. Further investigation is needed to reduce the
   2146  * saved/restored registers even further, by following the same 3 criteria.
   2147  */
   2148 static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv)
   2149 {
   2150 	struct vlv_s0ix_state *s = dev_priv->vlv_s0ix_state;
   2151 	int i;
   2152 
   2153 	if (!s)
   2154 		return;
   2155 
   2156 	/* GAM 0x4000-0x4770 */
   2157 	s->wr_watermark		= I915_READ(GEN7_WR_WATERMARK);
   2158 	s->gfx_prio_ctrl	= I915_READ(GEN7_GFX_PRIO_CTRL);
   2159 	s->arb_mode		= I915_READ(ARB_MODE);
   2160 	s->gfx_pend_tlb0	= I915_READ(GEN7_GFX_PEND_TLB0);
   2161 	s->gfx_pend_tlb1	= I915_READ(GEN7_GFX_PEND_TLB1);
   2162 
   2163 	for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
   2164 		s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i));
   2165 
   2166 	s->media_max_req_count	= I915_READ(GEN7_MEDIA_MAX_REQ_COUNT);
   2167 	s->gfx_max_req_count	= I915_READ(GEN7_GFX_MAX_REQ_COUNT);
   2168 
   2169 	s->render_hwsp		= I915_READ(RENDER_HWS_PGA_GEN7);
   2170 	s->ecochk		= I915_READ(GAM_ECOCHK);
   2171 	s->bsd_hwsp		= I915_READ(BSD_HWS_PGA_GEN7);
   2172 	s->blt_hwsp		= I915_READ(BLT_HWS_PGA_GEN7);
   2173 
   2174 	s->tlb_rd_addr		= I915_READ(GEN7_TLB_RD_ADDR);
   2175 
   2176 	/* MBC 0x9024-0x91D0, 0x8500 */
   2177 	s->g3dctl		= I915_READ(VLV_G3DCTL);
   2178 	s->gsckgctl		= I915_READ(VLV_GSCKGCTL);
   2179 	s->mbctl		= I915_READ(GEN6_MBCTL);
   2180 
   2181 	/* GCP 0x9400-0x9424, 0x8100-0x810C */
   2182 	s->ucgctl1		= I915_READ(GEN6_UCGCTL1);
   2183 	s->ucgctl3		= I915_READ(GEN6_UCGCTL3);
   2184 	s->rcgctl1		= I915_READ(GEN6_RCGCTL1);
   2185 	s->rcgctl2		= I915_READ(GEN6_RCGCTL2);
   2186 	s->rstctl		= I915_READ(GEN6_RSTCTL);
   2187 	s->misccpctl		= I915_READ(GEN7_MISCCPCTL);
   2188 
   2189 	/* GPM 0xA000-0xAA84, 0x8000-0x80FC */
   2190 	s->gfxpause		= I915_READ(GEN6_GFXPAUSE);
   2191 	s->rpdeuhwtc		= I915_READ(GEN6_RPDEUHWTC);
   2192 	s->rpdeuc		= I915_READ(GEN6_RPDEUC);
   2193 	s->ecobus		= I915_READ(ECOBUS);
   2194 	s->pwrdwnupctl		= I915_READ(VLV_PWRDWNUPCTL);
   2195 	s->rp_down_timeout	= I915_READ(GEN6_RP_DOWN_TIMEOUT);
   2196 	s->rp_deucsw		= I915_READ(GEN6_RPDEUCSW);
   2197 	s->rcubmabdtmr		= I915_READ(GEN6_RCUBMABDTMR);
   2198 	s->rcedata		= I915_READ(VLV_RCEDATA);
   2199 	s->spare2gh		= I915_READ(VLV_SPAREG2H);
   2200 
   2201 	/* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
   2202 	s->gt_imr		= I915_READ(GTIMR);
   2203 	s->gt_ier		= I915_READ(GTIER);
   2204 	s->pm_imr		= I915_READ(GEN6_PMIMR);
   2205 	s->pm_ier		= I915_READ(GEN6_PMIER);
   2206 
   2207 	for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
   2208 		s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i));
   2209 
   2210 	/* GT SA CZ domain, 0x100000-0x138124 */
   2211 	s->tilectl		= I915_READ(TILECTL);
   2212 	s->gt_fifoctl		= I915_READ(GTFIFOCTL);
   2213 	s->gtlc_wake_ctrl	= I915_READ(VLV_GTLC_WAKE_CTRL);
   2214 	s->gtlc_survive		= I915_READ(VLV_GTLC_SURVIVABILITY_REG);
   2215 	s->pmwgicz		= I915_READ(VLV_PMWGICZ);
   2216 
   2217 	/* Gunit-Display CZ domain, 0x182028-0x1821CF */
   2218 	s->gu_ctl0		= I915_READ(VLV_GU_CTL0);
   2219 	s->gu_ctl1		= I915_READ(VLV_GU_CTL1);
   2220 	s->pcbr			= I915_READ(VLV_PCBR);
   2221 	s->clock_gate_dis2	= I915_READ(VLV_GUNIT_CLOCK_GATE2);
   2222 
   2223 	/*
   2224 	 * Not saving any of:
   2225 	 * DFT,		0x9800-0x9EC0
   2226 	 * SARB,	0xB000-0xB1FC
   2227 	 * GAC,		0x5208-0x524C, 0x14000-0x14C000
   2228 	 * PCI CFG
   2229 	 */
   2230 }
   2231 
   2232 static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv)
   2233 {
   2234 	struct vlv_s0ix_state *s = dev_priv->vlv_s0ix_state;
   2235 	u32 val;
   2236 	int i;
   2237 
   2238 	if (!s)
   2239 		return;
   2240 
   2241 	/* GAM 0x4000-0x4770 */
   2242 	I915_WRITE(GEN7_WR_WATERMARK,	s->wr_watermark);
   2243 	I915_WRITE(GEN7_GFX_PRIO_CTRL,	s->gfx_prio_ctrl);
   2244 	I915_WRITE(ARB_MODE,		s->arb_mode | (0xffff << 16));
   2245 	I915_WRITE(GEN7_GFX_PEND_TLB0,	s->gfx_pend_tlb0);
   2246 	I915_WRITE(GEN7_GFX_PEND_TLB1,	s->gfx_pend_tlb1);
   2247 
   2248 	for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
   2249 		I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]);
   2250 
   2251 	I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count);
   2252 	I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count);
   2253 
   2254 	I915_WRITE(RENDER_HWS_PGA_GEN7,	s->render_hwsp);
   2255 	I915_WRITE(GAM_ECOCHK,		s->ecochk);
   2256 	I915_WRITE(BSD_HWS_PGA_GEN7,	s->bsd_hwsp);
   2257 	I915_WRITE(BLT_HWS_PGA_GEN7,	s->blt_hwsp);
   2258 
   2259 	I915_WRITE(GEN7_TLB_RD_ADDR,	s->tlb_rd_addr);
   2260 
   2261 	/* MBC 0x9024-0x91D0, 0x8500 */
   2262 	I915_WRITE(VLV_G3DCTL,		s->g3dctl);
   2263 	I915_WRITE(VLV_GSCKGCTL,	s->gsckgctl);
   2264 	I915_WRITE(GEN6_MBCTL,		s->mbctl);
   2265 
   2266 	/* GCP 0x9400-0x9424, 0x8100-0x810C */
   2267 	I915_WRITE(GEN6_UCGCTL1,	s->ucgctl1);
   2268 	I915_WRITE(GEN6_UCGCTL3,	s->ucgctl3);
   2269 	I915_WRITE(GEN6_RCGCTL1,	s->rcgctl1);
   2270 	I915_WRITE(GEN6_RCGCTL2,	s->rcgctl2);
   2271 	I915_WRITE(GEN6_RSTCTL,		s->rstctl);
   2272 	I915_WRITE(GEN7_MISCCPCTL,	s->misccpctl);
   2273 
   2274 	/* GPM 0xA000-0xAA84, 0x8000-0x80FC */
   2275 	I915_WRITE(GEN6_GFXPAUSE,	s->gfxpause);
   2276 	I915_WRITE(GEN6_RPDEUHWTC,	s->rpdeuhwtc);
   2277 	I915_WRITE(GEN6_RPDEUC,		s->rpdeuc);
   2278 	I915_WRITE(ECOBUS,		s->ecobus);
   2279 	I915_WRITE(VLV_PWRDWNUPCTL,	s->pwrdwnupctl);
   2280 	I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout);
   2281 	I915_WRITE(GEN6_RPDEUCSW,	s->rp_deucsw);
   2282 	I915_WRITE(GEN6_RCUBMABDTMR,	s->rcubmabdtmr);
   2283 	I915_WRITE(VLV_RCEDATA,		s->rcedata);
   2284 	I915_WRITE(VLV_SPAREG2H,	s->spare2gh);
   2285 
   2286 	/* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
   2287 	I915_WRITE(GTIMR,		s->gt_imr);
   2288 	I915_WRITE(GTIER,		s->gt_ier);
   2289 	I915_WRITE(GEN6_PMIMR,		s->pm_imr);
   2290 	I915_WRITE(GEN6_PMIER,		s->pm_ier);
   2291 
   2292 	for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
   2293 		I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]);
   2294 
   2295 	/* GT SA CZ domain, 0x100000-0x138124 */
   2296 	I915_WRITE(TILECTL,			s->tilectl);
   2297 	I915_WRITE(GTFIFOCTL,			s->gt_fifoctl);
   2298 	/*
   2299 	 * Preserve the GT allow wake and GFX force clock bit, they are not
   2300 	 * be restored, as they are used to control the s0ix suspend/resume
   2301 	 * sequence by the caller.
   2302 	 */
   2303 	val = I915_READ(VLV_GTLC_WAKE_CTRL);
   2304 	val &= VLV_GTLC_ALLOWWAKEREQ;
   2305 	val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ;
   2306 	I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
   2307 
   2308 	val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
   2309 	val &= VLV_GFX_CLK_FORCE_ON_BIT;
   2310 	val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT;
   2311 	I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
   2312 
   2313 	I915_WRITE(VLV_PMWGICZ,			s->pmwgicz);
   2314 
   2315 	/* Gunit-Display CZ domain, 0x182028-0x1821CF */
   2316 	I915_WRITE(VLV_GU_CTL0,			s->gu_ctl0);
   2317 	I915_WRITE(VLV_GU_CTL1,			s->gu_ctl1);
   2318 	I915_WRITE(VLV_PCBR,			s->pcbr);
   2319 	I915_WRITE(VLV_GUNIT_CLOCK_GATE2,	s->clock_gate_dis2);
   2320 }
   2321 
   2322 static int vlv_wait_for_pw_status(struct drm_i915_private *i915,
   2323 				  u32 mask, u32 val)
   2324 {
   2325 	i915_reg_t reg = VLV_GTLC_PW_STATUS;
   2326 	u32 reg_value;
   2327 	int ret;
   2328 
   2329 	/* The HW does not like us polling for PW_STATUS frequently, so
   2330 	 * use the sleeping loop rather than risk the busy spin within
   2331 	 * intel_wait_for_register().
   2332 	 *
   2333 	 * Transitioning between RC6 states should be at most 2ms (see
   2334 	 * valleyview_enable_rps) so use a 3ms timeout.
   2335 	 */
   2336 	ret = wait_for(((reg_value =
   2337 			 intel_uncore_read_notrace(&i915->uncore, reg)) & mask)
   2338 		       == val, 3);
   2339 
   2340 	/* just trace the final value */
   2341 	trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true);
   2342 
   2343 	return ret;
   2344 }
   2345 
   2346 int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on)
   2347 {
   2348 	u32 val;
   2349 	int err;
   2350 
   2351 	val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
   2352 	val &= ~VLV_GFX_CLK_FORCE_ON_BIT;
   2353 	if (force_on)
   2354 		val |= VLV_GFX_CLK_FORCE_ON_BIT;
   2355 	I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
   2356 
   2357 	if (!force_on)
   2358 		return 0;
   2359 
   2360 	err = intel_wait_for_register(&dev_priv->uncore,
   2361 				      VLV_GTLC_SURVIVABILITY_REG,
   2362 				      VLV_GFX_CLK_STATUS_BIT,
   2363 				      VLV_GFX_CLK_STATUS_BIT,
   2364 				      20);
   2365 	if (err)
   2366 		DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n",
   2367 			  I915_READ(VLV_GTLC_SURVIVABILITY_REG));
   2368 
   2369 	return err;
   2370 }
   2371 
   2372 static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow)
   2373 {
   2374 	u32 mask;
   2375 	u32 val;
   2376 	int err;
   2377 
   2378 	val = I915_READ(VLV_GTLC_WAKE_CTRL);
   2379 	val &= ~VLV_GTLC_ALLOWWAKEREQ;
   2380 	if (allow)
   2381 		val |= VLV_GTLC_ALLOWWAKEREQ;
   2382 	I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
   2383 	POSTING_READ(VLV_GTLC_WAKE_CTRL);
   2384 
   2385 	mask = VLV_GTLC_ALLOWWAKEACK;
   2386 	val = allow ? mask : 0;
   2387 
   2388 	err = vlv_wait_for_pw_status(dev_priv, mask, val);
   2389 	if (err)
   2390 		DRM_ERROR("timeout disabling GT waking\n");
   2391 
   2392 	return err;
   2393 }
   2394 
   2395 static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
   2396 				  bool wait_for_on)
   2397 {
   2398 	u32 mask;
   2399 	u32 val;
   2400 
   2401 	mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;
   2402 	val = wait_for_on ? mask : 0;
   2403 
   2404 	/*
   2405 	 * RC6 transitioning can be delayed up to 2 msec (see
   2406 	 * valleyview_enable_rps), use 3 msec for safety.
   2407 	 *
   2408 	 * This can fail to turn off the rc6 if the GPU is stuck after a failed
   2409 	 * reset and we are trying to force the machine to sleep.
   2410 	 */
   2411 	if (vlv_wait_for_pw_status(dev_priv, mask, val))
   2412 		DRM_DEBUG_DRIVER("timeout waiting for GT wells to go %s\n",
   2413 				 onoff(wait_for_on));
   2414 }
   2415 
   2416 static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv)
   2417 {
   2418 	if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR))
   2419 		return;
   2420 
   2421 	DRM_DEBUG_DRIVER("GT register access while GT waking disabled\n");
   2422 	I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
   2423 }
   2424 
   2425 static int vlv_suspend_complete(struct drm_i915_private *dev_priv)
   2426 {
   2427 	u32 mask;
   2428 	int err;
   2429 
   2430 	/*
   2431 	 * Bspec defines the following GT well on flags as debug only, so
   2432 	 * don't treat them as hard failures.
   2433 	 */
   2434 	vlv_wait_for_gt_wells(dev_priv, false);
   2435 
   2436 	mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS;
   2437 	WARN_ON((I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask);
   2438 
   2439 	vlv_check_no_gt_access(dev_priv);
   2440 
   2441 	err = vlv_force_gfx_clock(dev_priv, true);
   2442 	if (err)
   2443 		goto err1;
   2444 
   2445 	err = vlv_allow_gt_wake(dev_priv, false);
   2446 	if (err)
   2447 		goto err2;
   2448 
   2449 	vlv_save_gunit_s0ix_state(dev_priv);
   2450 
   2451 	err = vlv_force_gfx_clock(dev_priv, false);
   2452 	if (err)
   2453 		goto err2;
   2454 
   2455 	return 0;
   2456 
   2457 err2:
   2458 	/* For safety always re-enable waking and disable gfx clock forcing */
   2459 	vlv_allow_gt_wake(dev_priv, true);
   2460 err1:
   2461 	vlv_force_gfx_clock(dev_priv, false);
   2462 
   2463 	return err;
   2464 }
   2465 
   2466 static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
   2467 				bool rpm_resume)
   2468 {
   2469 	int err;
   2470 	int ret;
   2471 
   2472 	/*
   2473 	 * If any of the steps fail just try to continue, that's the best we
   2474 	 * can do at this point. Return the first error code (which will also
   2475 	 * leave RPM permanently disabled).
   2476 	 */
   2477 	ret = vlv_force_gfx_clock(dev_priv, true);
   2478 
   2479 	vlv_restore_gunit_s0ix_state(dev_priv);
   2480 
   2481 	err = vlv_allow_gt_wake(dev_priv, true);
   2482 	if (!ret)
   2483 		ret = err;
   2484 
   2485 	err = vlv_force_gfx_clock(dev_priv, false);
   2486 	if (!ret)
   2487 		ret = err;
   2488 
   2489 	vlv_check_no_gt_access(dev_priv);
   2490 
   2491 	if (rpm_resume)
   2492 		intel_init_clock_gating(dev_priv);
   2493 
   2494 	return ret;
   2495 }
   2496 
   2497 static int intel_runtime_suspend(struct device *kdev)
   2498 {
   2499 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
   2500 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
   2501 	int ret = 0;
   2502 
   2503 	if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv)))
   2504 		return -ENODEV;
   2505 
   2506 	DRM_DEBUG_KMS("Suspending device\n");
   2507 
   2508 	disable_rpm_wakeref_asserts(rpm);
   2509 
   2510 	/*
   2511 	 * We are safe here against re-faults, since the fault handler takes
   2512 	 * an RPM reference.
   2513 	 */
   2514 	i915_gem_runtime_suspend(dev_priv);
   2515 
   2516 	intel_gt_runtime_suspend(&dev_priv->gt);
   2517 
   2518 	intel_runtime_pm_disable_interrupts(dev_priv);
   2519 
   2520 	intel_uncore_suspend(&dev_priv->uncore);
   2521 
   2522 	intel_display_power_suspend(dev_priv);
   2523 
   2524 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
   2525 		ret = vlv_suspend_complete(dev_priv);
   2526 
   2527 	if (ret) {
   2528 		DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret);
   2529 		intel_uncore_runtime_resume(&dev_priv->uncore);
   2530 
   2531 		intel_runtime_pm_enable_interrupts(dev_priv);
   2532 
   2533 		intel_gt_runtime_resume(&dev_priv->gt);
   2534 
   2535 		i915_gem_restore_fences(&dev_priv->ggtt);
   2536 
   2537 		enable_rpm_wakeref_asserts(rpm);
   2538 
   2539 		return ret;
   2540 	}
   2541 
   2542 	enable_rpm_wakeref_asserts(rpm);
   2543 	intel_runtime_pm_driver_release(rpm);
   2544 
   2545 	if (intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore))
   2546 		DRM_ERROR("Unclaimed access detected prior to suspending\n");
   2547 
   2548 	rpm->suspended = true;
   2549 
   2550 	/*
   2551 	 * FIXME: We really should find a document that references the arguments
   2552 	 * used below!
   2553 	 */
   2554 	if (IS_BROADWELL(dev_priv)) {
   2555 		/*
   2556 		 * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
   2557 		 * being detected, and the call we do at intel_runtime_resume()
   2558 		 * won't be able to restore them. Since PCI_D3hot matches the
   2559 		 * actual specification and appears to be working, use it.
   2560 		 */
   2561 		intel_opregion_notify_adapter(dev_priv, PCI_D3hot);
   2562 	} else {
   2563 		/*
   2564 		 * current versions of firmware which depend on this opregion
   2565 		 * notification have repurposed the D1 definition to mean
   2566 		 * "runtime suspended" vs. what you would normally expect (D3)
   2567 		 * to distinguish it from notifications that might be sent via
   2568 		 * the suspend path.
   2569 		 */
   2570 		intel_opregion_notify_adapter(dev_priv, PCI_D1);
   2571 	}
   2572 
   2573 	assert_forcewakes_inactive(&dev_priv->uncore);
   2574 
   2575 	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
   2576 		intel_hpd_poll_init(dev_priv);
   2577 
   2578 	DRM_DEBUG_KMS("Device suspended\n");
   2579 	return 0;
   2580 }
   2581 
   2582 static int intel_runtime_resume(struct device *kdev)
   2583 {
   2584 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
   2585 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
   2586 	int ret = 0;
   2587 
   2588 	if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv)))
   2589 		return -ENODEV;
   2590 
   2591 	DRM_DEBUG_KMS("Resuming device\n");
   2592 
   2593 	WARN_ON_ONCE(atomic_read(&rpm->wakeref_count));
   2594 	disable_rpm_wakeref_asserts(rpm);
   2595 
   2596 	intel_opregion_notify_adapter(dev_priv, PCI_D0);
   2597 	rpm->suspended = false;
   2598 	if (intel_uncore_unclaimed_mmio(&dev_priv->uncore))
   2599 		DRM_DEBUG_DRIVER("Unclaimed access during suspend, bios?\n");
   2600 
   2601 	intel_display_power_resume(dev_priv);
   2602 
   2603 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
   2604 		ret = vlv_resume_prepare(dev_priv, true);
   2605 
   2606 	intel_uncore_runtime_resume(&dev_priv->uncore);
   2607 
   2608 	intel_runtime_pm_enable_interrupts(dev_priv);
   2609 
   2610 	/*
   2611 	 * No point of rolling back things in case of an error, as the best
   2612 	 * we can do is to hope that things will still work (and disable RPM).
   2613 	 */
   2614 	intel_gt_runtime_resume(&dev_priv->gt);
   2615 	i915_gem_restore_fences(&dev_priv->ggtt);
   2616 
   2617 	/*
   2618 	 * On VLV/CHV display interrupts are part of the display
   2619 	 * power well, so hpd is reinitialized from there. For
   2620 	 * everyone else do it here.
   2621 	 */
   2622 	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
   2623 		intel_hpd_init(dev_priv);
   2624 
   2625 	intel_enable_ipc(dev_priv);
   2626 
   2627 	enable_rpm_wakeref_asserts(rpm);
   2628 
   2629 	if (ret)
   2630 		DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret);
   2631 	else
   2632 		DRM_DEBUG_KMS("Device resumed\n");
   2633 
   2634 	return ret;
   2635 }
   2636 
   2637 const struct dev_pm_ops i915_pm_ops = {
   2638 	/*
   2639 	 * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND,
   2640 	 * PMSG_RESUME]
   2641 	 */
   2642 	.prepare = i915_pm_prepare,
   2643 	.suspend = i915_pm_suspend,
   2644 	.suspend_late = i915_pm_suspend_late,
   2645 	.resume_early = i915_pm_resume_early,
   2646 	.resume = i915_pm_resume,
   2647 
   2648 	/*
   2649 	 * S4 event handlers
   2650 	 * @freeze, @freeze_late    : called (1) before creating the
   2651 	 *                            hibernation image [PMSG_FREEZE] and
   2652 	 *                            (2) after rebooting, before restoring
   2653 	 *                            the image [PMSG_QUIESCE]
   2654 	 * @thaw, @thaw_early       : called (1) after creating the hibernation
   2655 	 *                            image, before writing it [PMSG_THAW]
   2656 	 *                            and (2) after failing to create or
   2657 	 *                            restore the image [PMSG_RECOVER]
   2658 	 * @poweroff, @poweroff_late: called after writing the hibernation
   2659 	 *                            image, before rebooting [PMSG_HIBERNATE]
   2660 	 * @restore, @restore_early : called after rebooting and restoring the
   2661 	 *                            hibernation image [PMSG_RESTORE]
   2662 	 */
   2663 	.freeze = i915_pm_freeze,
   2664 	.freeze_late = i915_pm_freeze_late,
   2665 	.thaw_early = i915_pm_thaw_early,
   2666 	.thaw = i915_pm_thaw,
   2667 	.poweroff = i915_pm_suspend,
   2668 	.poweroff_late = i915_pm_poweroff_late,
   2669 	.restore_early = i915_pm_restore_early,
   2670 	.restore = i915_pm_restore,
   2671 
   2672 	/* S0ix (via runtime suspend) event handlers */
   2673 	.runtime_suspend = intel_runtime_suspend,
   2674 	.runtime_resume = intel_runtime_resume,
   2675 };
   2676 
   2677 static const struct file_operations i915_driver_fops = {
   2678 	.owner = THIS_MODULE,
   2679 	.open = drm_open,
   2680 	.release = drm_release,
   2681 	.unlocked_ioctl = drm_ioctl,
   2682 	.mmap = i915_gem_mmap,
   2683 	.poll = drm_poll,
   2684 	.read = drm_read,
   2685 	.compat_ioctl = i915_compat_ioctl,
   2686 	.llseek = noop_llseek,
   2687 };
   2688 
   2689 static int
   2690 i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data,
   2691 			  struct drm_file *file)
   2692 {
   2693 	return -ENODEV;
   2694 }
   2695 
   2696 static const struct drm_ioctl_desc i915_ioctls[] = {
   2697 	DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
   2698 	DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH),
   2699 	DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH),
   2700 	DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH),
   2701 	DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH),
   2702 	DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH),
   2703 	DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam_ioctl, DRM_RENDER_ALLOW),
   2704 	DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
   2705 	DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH),
   2706 	DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH),
   2707 	DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
   2708 	DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH),
   2709 	DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
   2710 	DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
   2711 	DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE,  drm_noop, DRM_AUTH),
   2712 	DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH),
   2713 	DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
   2714 	DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
   2715 	DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer_ioctl, DRM_AUTH),
   2716 	DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2_WR, i915_gem_execbuffer2_ioctl, DRM_RENDER_ALLOW),
   2717 	DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
   2718 	DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
   2719 	DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_RENDER_ALLOW),
   2720 	DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW),
   2721 	DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW),
   2722 	DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_RENDER_ALLOW),
   2723 	DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
   2724 	DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
   2725 	DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW),
   2726 	DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW),
   2727 	DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW),
   2728 	DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW),
   2729 	DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_OFFSET, i915_gem_mmap_offset_ioctl, DRM_RENDER_ALLOW),
   2730 	DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW),
   2731 	DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW),
   2732 	DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling_ioctl, DRM_RENDER_ALLOW),
   2733 	DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling_ioctl, DRM_RENDER_ALLOW),
   2734 	DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW),
   2735 	DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id_ioctl, 0),
   2736 	DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW),
   2737 	DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER),
   2738 	DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER),
   2739 	DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey_ioctl, DRM_MASTER),
   2740 	DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER),
   2741 	DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_RENDER_ALLOW),
   2742 	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE_EXT, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW),
   2743 	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW),
   2744 	DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW),
   2745 	DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW),
   2746 	DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW),
   2747 	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW),
   2748 	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW),
   2749 	DRM_IOCTL_DEF_DRV(I915_PERF_OPEN, i915_perf_open_ioctl, DRM_RENDER_ALLOW),
   2750 	DRM_IOCTL_DEF_DRV(I915_PERF_ADD_CONFIG, i915_perf_add_config_ioctl, DRM_RENDER_ALLOW),
   2751 	DRM_IOCTL_DEF_DRV(I915_PERF_REMOVE_CONFIG, i915_perf_remove_config_ioctl, DRM_RENDER_ALLOW),
   2752 	DRM_IOCTL_DEF_DRV(I915_QUERY, i915_query_ioctl, DRM_RENDER_ALLOW),
   2753 	DRM_IOCTL_DEF_DRV(I915_GEM_VM_CREATE, i915_gem_vm_create_ioctl, DRM_RENDER_ALLOW),
   2754 	DRM_IOCTL_DEF_DRV(I915_GEM_VM_DESTROY, i915_gem_vm_destroy_ioctl, DRM_RENDER_ALLOW),
   2755 };
   2756 
   2757 static struct drm_driver driver = {
   2758 	/* Don't use MTRRs here; the Xserver or userspace app should
   2759 	 * deal with them for Intel hardware.
   2760 	 */
   2761 	.driver_features =
   2762 	    DRIVER_GEM |
   2763 	    DRIVER_RENDER | DRIVER_MODESET | DRIVER_ATOMIC | DRIVER_SYNCOBJ,
   2764 	.release = i915_driver_release,
   2765 	.open = i915_driver_open,
   2766 	.lastclose = i915_driver_lastclose,
   2767 	.postclose = i915_driver_postclose,
   2768 
   2769 	.gem_close_object = i915_gem_close_object,
   2770 	.gem_free_object_unlocked = i915_gem_free_object,
   2771 
   2772 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
   2773 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
   2774 	.gem_prime_export = i915_gem_prime_export,
   2775 	.gem_prime_import = i915_gem_prime_import,
   2776 
   2777 	.get_vblank_timestamp = drm_calc_vbltimestamp_from_scanoutpos,
   2778 	.get_scanout_position = i915_get_crtc_scanoutpos,
   2779 
   2780 	.dumb_create = i915_gem_dumb_create,
   2781 	.dumb_map_offset = i915_gem_dumb_mmap_offset,
   2782 
   2783 	.ioctls = i915_ioctls,
   2784 	.num_ioctls = ARRAY_SIZE(i915_ioctls),
   2785 	.fops = &i915_driver_fops,
   2786 	.name = DRIVER_NAME,
   2787 	.desc = DRIVER_DESC,
   2788 	.date = DRIVER_DATE,
   2789 	.major = DRIVER_MAJOR,
   2790 	.minor = DRIVER_MINOR,
   2791 	.patchlevel = DRIVER_PATCHLEVEL,
   2792 };
   2793