Home | History | Annotate | Line # | Download | only in i915
i915_drv.c revision 1.9
      1 /*	$NetBSD: i915_drv.c,v 1.9 2018/08/27 07:03:25 riastradh Exp $	*/
      2 
      3 /* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*-
      4  */
      5 /*
      6  *
      7  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
      8  * All Rights Reserved.
      9  *
     10  * Permission is hereby granted, free of charge, to any person obtaining a
     11  * copy of this software and associated documentation files (the
     12  * "Software"), to deal in the Software without restriction, including
     13  * without limitation the rights to use, copy, modify, merge, publish,
     14  * distribute, sub license, and/or sell copies of the Software, and to
     15  * permit persons to whom the Software is furnished to do so, subject to
     16  * the following conditions:
     17  *
     18  * The above copyright notice and this permission notice (including the
     19  * next paragraph) shall be included in all copies or substantial portions
     20  * of the Software.
     21  *
     22  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
     23  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
     24  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
     25  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
     26  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
     27  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
     28  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
     29  *
     30  */
     31 
     32 #include <sys/cdefs.h>
     33 __KERNEL_RCSID(0, "$NetBSD: i915_drv.c,v 1.9 2018/08/27 07:03:25 riastradh Exp $");
     34 
     35 #include <linux/device.h>
     36 #include <linux/acpi.h>
     37 #include <linux/moduleparam.h>
     38 #include <linux/time.h>
     39 #include <drm/drmP.h>
     40 #include <drm/i915_drm.h>
     41 #include "i915_drv.h"
     42 #include "i915_trace.h"
     43 #include "intel_drv.h"
     44 
     45 #include <linux/console.h>
     46 #include <linux/module.h>
     47 #include <linux/pm_runtime.h>
     48 #include <drm/drm_crtc_helper.h>
     49 
     50 static struct drm_driver driver;
     51 
     52 #ifdef __NetBSD__
     53 /* XXX Kludge to expose this to NetBSD driver attachment goop.  */
     54 struct drm_driver *const i915_drm_driver = &driver;
     55 #endif
     56 
     57 #define GEN_DEFAULT_PIPEOFFSETS \
     58 	.pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \
     59 			  PIPE_C_OFFSET, PIPE_EDP_OFFSET }, \
     60 	.trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \
     61 			   TRANSCODER_C_OFFSET, TRANSCODER_EDP_OFFSET }, \
     62 	.palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET }
     63 
     64 #define GEN_CHV_PIPEOFFSETS \
     65 	.pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \
     66 			  CHV_PIPE_C_OFFSET }, \
     67 	.trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \
     68 			   CHV_TRANSCODER_C_OFFSET, }, \
     69 	.palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET, \
     70 			     CHV_PALETTE_C_OFFSET }
     71 
     72 #define CURSOR_OFFSETS \
     73 	.cursor_offsets = { CURSOR_A_OFFSET, CURSOR_B_OFFSET, CHV_CURSOR_C_OFFSET }
     74 
     75 #define IVB_CURSOR_OFFSETS \
     76 	.cursor_offsets = { CURSOR_A_OFFSET, IVB_CURSOR_B_OFFSET, IVB_CURSOR_C_OFFSET }
     77 
     78 static const struct intel_device_info intel_i830_info = {
     79 	.gen = 2, .is_mobile = 1, .cursor_needs_physical = 1, .num_pipes = 2,
     80 	.has_overlay = 1, .overlay_needs_physical = 1,
     81 	.ring_mask = RENDER_RING,
     82 	GEN_DEFAULT_PIPEOFFSETS,
     83 	CURSOR_OFFSETS,
     84 };
     85 
     86 static const struct intel_device_info intel_845g_info = {
     87 	.gen = 2, .num_pipes = 1,
     88 	.has_overlay = 1, .overlay_needs_physical = 1,
     89 	.ring_mask = RENDER_RING,
     90 	GEN_DEFAULT_PIPEOFFSETS,
     91 	CURSOR_OFFSETS,
     92 };
     93 
     94 static const struct intel_device_info intel_i85x_info = {
     95 	.gen = 2, .is_i85x = 1, .is_mobile = 1, .num_pipes = 2,
     96 	.cursor_needs_physical = 1,
     97 	.has_overlay = 1, .overlay_needs_physical = 1,
     98 	.has_fbc = 1,
     99 	.ring_mask = RENDER_RING,
    100 	GEN_DEFAULT_PIPEOFFSETS,
    101 	CURSOR_OFFSETS,
    102 };
    103 
    104 static const struct intel_device_info intel_i865g_info = {
    105 	.gen = 2, .num_pipes = 1,
    106 	.has_overlay = 1, .overlay_needs_physical = 1,
    107 	.ring_mask = RENDER_RING,
    108 	GEN_DEFAULT_PIPEOFFSETS,
    109 	CURSOR_OFFSETS,
    110 };
    111 
    112 static const struct intel_device_info intel_i915g_info = {
    113 	.gen = 3, .is_i915g = 1, .cursor_needs_physical = 1, .num_pipes = 2,
    114 	.has_overlay = 1, .overlay_needs_physical = 1,
    115 	.ring_mask = RENDER_RING,
    116 	GEN_DEFAULT_PIPEOFFSETS,
    117 	CURSOR_OFFSETS,
    118 };
    119 static const struct intel_device_info intel_i915gm_info = {
    120 	.gen = 3, .is_mobile = 1, .num_pipes = 2,
    121 	.cursor_needs_physical = 1,
    122 	.has_overlay = 1, .overlay_needs_physical = 1,
    123 	.supports_tv = 1,
    124 	.has_fbc = 1,
    125 	.ring_mask = RENDER_RING,
    126 	GEN_DEFAULT_PIPEOFFSETS,
    127 	CURSOR_OFFSETS,
    128 };
    129 static const struct intel_device_info intel_i945g_info = {
    130 	.gen = 3, .has_hotplug = 1, .cursor_needs_physical = 1, .num_pipes = 2,
    131 	.has_overlay = 1, .overlay_needs_physical = 1,
    132 	.ring_mask = RENDER_RING,
    133 	GEN_DEFAULT_PIPEOFFSETS,
    134 	CURSOR_OFFSETS,
    135 };
    136 static const struct intel_device_info intel_i945gm_info = {
    137 	.gen = 3, .is_i945gm = 1, .is_mobile = 1, .num_pipes = 2,
    138 	.has_hotplug = 1, .cursor_needs_physical = 1,
    139 	.has_overlay = 1, .overlay_needs_physical = 1,
    140 	.supports_tv = 1,
    141 	.has_fbc = 1,
    142 	.ring_mask = RENDER_RING,
    143 	GEN_DEFAULT_PIPEOFFSETS,
    144 	CURSOR_OFFSETS,
    145 };
    146 
    147 static const struct intel_device_info intel_i965g_info = {
    148 	.gen = 4, .is_broadwater = 1, .num_pipes = 2,
    149 	.has_hotplug = 1,
    150 	.has_overlay = 1,
    151 	.ring_mask = RENDER_RING,
    152 	GEN_DEFAULT_PIPEOFFSETS,
    153 	CURSOR_OFFSETS,
    154 };
    155 
    156 static const struct intel_device_info intel_i965gm_info = {
    157 	.gen = 4, .is_crestline = 1, .num_pipes = 2,
    158 	.is_mobile = 1, .has_fbc = 1, .has_hotplug = 1,
    159 	.has_overlay = 1,
    160 	.supports_tv = 1,
    161 	.ring_mask = RENDER_RING,
    162 	GEN_DEFAULT_PIPEOFFSETS,
    163 	CURSOR_OFFSETS,
    164 };
    165 
    166 static const struct intel_device_info intel_g33_info = {
    167 	.gen = 3, .is_g33 = 1, .num_pipes = 2,
    168 	.need_gfx_hws = 1, .has_hotplug = 1,
    169 	.has_overlay = 1,
    170 	.ring_mask = RENDER_RING,
    171 	GEN_DEFAULT_PIPEOFFSETS,
    172 	CURSOR_OFFSETS,
    173 };
    174 
    175 static const struct intel_device_info intel_g45_info = {
    176 	.gen = 4, .is_g4x = 1, .need_gfx_hws = 1, .num_pipes = 2,
    177 	.has_pipe_cxsr = 1, .has_hotplug = 1,
    178 	.ring_mask = RENDER_RING | BSD_RING,
    179 	GEN_DEFAULT_PIPEOFFSETS,
    180 	CURSOR_OFFSETS,
    181 };
    182 
    183 static const struct intel_device_info intel_gm45_info = {
    184 	.gen = 4, .is_g4x = 1, .num_pipes = 2,
    185 	.is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1,
    186 	.has_pipe_cxsr = 1, .has_hotplug = 1,
    187 	.supports_tv = 1,
    188 	.ring_mask = RENDER_RING | BSD_RING,
    189 	GEN_DEFAULT_PIPEOFFSETS,
    190 	CURSOR_OFFSETS,
    191 };
    192 
    193 static const struct intel_device_info intel_pineview_info = {
    194 	.gen = 3, .is_g33 = 1, .is_pineview = 1, .is_mobile = 1, .num_pipes = 2,
    195 	.need_gfx_hws = 1, .has_hotplug = 1,
    196 	.has_overlay = 1,
    197 	GEN_DEFAULT_PIPEOFFSETS,
    198 	CURSOR_OFFSETS,
    199 };
    200 
    201 static const struct intel_device_info intel_ironlake_d_info = {
    202 	.gen = 5, .num_pipes = 2,
    203 	.need_gfx_hws = 1, .has_hotplug = 1,
    204 	.ring_mask = RENDER_RING | BSD_RING,
    205 	GEN_DEFAULT_PIPEOFFSETS,
    206 	CURSOR_OFFSETS,
    207 };
    208 
    209 static const struct intel_device_info intel_ironlake_m_info = {
    210 	.gen = 5, .is_mobile = 1, .num_pipes = 2,
    211 	.need_gfx_hws = 1, .has_hotplug = 1,
    212 	.has_fbc = 1,
    213 	.ring_mask = RENDER_RING | BSD_RING,
    214 	GEN_DEFAULT_PIPEOFFSETS,
    215 	CURSOR_OFFSETS,
    216 };
    217 
    218 static const struct intel_device_info intel_sandybridge_d_info = {
    219 	.gen = 6, .num_pipes = 2,
    220 	.need_gfx_hws = 1, .has_hotplug = 1,
    221 	.has_fbc = 1,
    222 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING,
    223 	.has_llc = 1,
    224 	GEN_DEFAULT_PIPEOFFSETS,
    225 	CURSOR_OFFSETS,
    226 };
    227 
    228 static const struct intel_device_info intel_sandybridge_m_info = {
    229 	.gen = 6, .is_mobile = 1, .num_pipes = 2,
    230 	.need_gfx_hws = 1, .has_hotplug = 1,
    231 	.has_fbc = 1,
    232 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING,
    233 	.has_llc = 1,
    234 	GEN_DEFAULT_PIPEOFFSETS,
    235 	CURSOR_OFFSETS,
    236 };
    237 
    238 #define GEN7_FEATURES  \
    239 	.gen = 7, .num_pipes = 3, \
    240 	.need_gfx_hws = 1, .has_hotplug = 1, \
    241 	.has_fbc = 1, \
    242 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING, \
    243 	.has_llc = 1
    244 
    245 static const struct intel_device_info intel_ivybridge_d_info = {
    246 	GEN7_FEATURES,
    247 	.is_ivybridge = 1,
    248 	GEN_DEFAULT_PIPEOFFSETS,
    249 	IVB_CURSOR_OFFSETS,
    250 };
    251 
    252 static const struct intel_device_info intel_ivybridge_m_info = {
    253 	GEN7_FEATURES,
    254 	.is_ivybridge = 1,
    255 	.is_mobile = 1,
    256 	GEN_DEFAULT_PIPEOFFSETS,
    257 	IVB_CURSOR_OFFSETS,
    258 };
    259 
    260 static const struct intel_device_info intel_ivybridge_q_info = {
    261 	GEN7_FEATURES,
    262 	.is_ivybridge = 1,
    263 	.num_pipes = 0, /* legal, last one wins */
    264 	GEN_DEFAULT_PIPEOFFSETS,
    265 	IVB_CURSOR_OFFSETS,
    266 };
    267 
    268 static const struct intel_device_info intel_valleyview_m_info = {
    269 	GEN7_FEATURES,
    270 	.is_mobile = 1,
    271 	.num_pipes = 2,
    272 	.is_valleyview = 1,
    273 	.display_mmio_offset = VLV_DISPLAY_BASE,
    274 	.has_fbc = 0, /* legal, last one wins */
    275 	.has_llc = 0, /* legal, last one wins */
    276 	GEN_DEFAULT_PIPEOFFSETS,
    277 	CURSOR_OFFSETS,
    278 };
    279 
    280 static const struct intel_device_info intel_valleyview_d_info = {
    281 	GEN7_FEATURES,
    282 	.num_pipes = 2,
    283 	.is_valleyview = 1,
    284 	.display_mmio_offset = VLV_DISPLAY_BASE,
    285 	.has_fbc = 0, /* legal, last one wins */
    286 	.has_llc = 0, /* legal, last one wins */
    287 	GEN_DEFAULT_PIPEOFFSETS,
    288 	CURSOR_OFFSETS,
    289 };
    290 
    291 static const struct intel_device_info intel_haswell_d_info = {
    292 	GEN7_FEATURES,
    293 	.is_haswell = 1,
    294 	.has_ddi = 1,
    295 	.has_fpga_dbg = 1,
    296 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
    297 	GEN_DEFAULT_PIPEOFFSETS,
    298 	IVB_CURSOR_OFFSETS,
    299 };
    300 
    301 static const struct intel_device_info intel_haswell_m_info = {
    302 	GEN7_FEATURES,
    303 	.is_haswell = 1,
    304 	.is_mobile = 1,
    305 	.has_ddi = 1,
    306 	.has_fpga_dbg = 1,
    307 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
    308 	GEN_DEFAULT_PIPEOFFSETS,
    309 	IVB_CURSOR_OFFSETS,
    310 };
    311 
    312 static const struct intel_device_info intel_broadwell_d_info = {
    313 	.gen = 8, .num_pipes = 3,
    314 	.need_gfx_hws = 1, .has_hotplug = 1,
    315 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
    316 	.has_llc = 1,
    317 	.has_ddi = 1,
    318 	.has_fpga_dbg = 1,
    319 	.has_fbc = 1,
    320 	GEN_DEFAULT_PIPEOFFSETS,
    321 	IVB_CURSOR_OFFSETS,
    322 };
    323 
    324 static const struct intel_device_info intel_broadwell_m_info = {
    325 	.gen = 8, .is_mobile = 1, .num_pipes = 3,
    326 	.need_gfx_hws = 1, .has_hotplug = 1,
    327 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
    328 	.has_llc = 1,
    329 	.has_ddi = 1,
    330 	.has_fpga_dbg = 1,
    331 	.has_fbc = 1,
    332 	GEN_DEFAULT_PIPEOFFSETS,
    333 	IVB_CURSOR_OFFSETS,
    334 };
    335 
    336 static const struct intel_device_info intel_broadwell_gt3d_info = {
    337 	.gen = 8, .num_pipes = 3,
    338 	.need_gfx_hws = 1, .has_hotplug = 1,
    339 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
    340 	.has_llc = 1,
    341 	.has_ddi = 1,
    342 	.has_fpga_dbg = 1,
    343 	.has_fbc = 1,
    344 	GEN_DEFAULT_PIPEOFFSETS,
    345 	IVB_CURSOR_OFFSETS,
    346 };
    347 
    348 static const struct intel_device_info intel_broadwell_gt3m_info = {
    349 	.gen = 8, .is_mobile = 1, .num_pipes = 3,
    350 	.need_gfx_hws = 1, .has_hotplug = 1,
    351 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
    352 	.has_llc = 1,
    353 	.has_ddi = 1,
    354 	.has_fpga_dbg = 1,
    355 	.has_fbc = 1,
    356 	GEN_DEFAULT_PIPEOFFSETS,
    357 	IVB_CURSOR_OFFSETS,
    358 };
    359 
    360 static const struct intel_device_info intel_cherryview_info = {
    361 	.gen = 8, .num_pipes = 3,
    362 	.need_gfx_hws = 1, .has_hotplug = 1,
    363 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
    364 	.is_valleyview = 1,
    365 	.display_mmio_offset = VLV_DISPLAY_BASE,
    366 	GEN_CHV_PIPEOFFSETS,
    367 	CURSOR_OFFSETS,
    368 };
    369 
    370 static const struct intel_device_info intel_skylake_info = {
    371 	.is_skylake = 1,
    372 	.gen = 9, .num_pipes = 3,
    373 	.need_gfx_hws = 1, .has_hotplug = 1,
    374 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
    375 	.has_llc = 1,
    376 	.has_ddi = 1,
    377 	.has_fpga_dbg = 1,
    378 	.has_fbc = 1,
    379 	GEN_DEFAULT_PIPEOFFSETS,
    380 	IVB_CURSOR_OFFSETS,
    381 };
    382 
    383 static const struct intel_device_info intel_skylake_gt3_info = {
    384 	.is_skylake = 1,
    385 	.gen = 9, .num_pipes = 3,
    386 	.need_gfx_hws = 1, .has_hotplug = 1,
    387 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
    388 	.has_llc = 1,
    389 	.has_ddi = 1,
    390 	.has_fpga_dbg = 1,
    391 	.has_fbc = 1,
    392 	GEN_DEFAULT_PIPEOFFSETS,
    393 	IVB_CURSOR_OFFSETS,
    394 };
    395 
    396 static const struct intel_device_info intel_broxton_info = {
    397 	.is_preliminary = 1,
    398 	.gen = 9,
    399 	.need_gfx_hws = 1, .has_hotplug = 1,
    400 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
    401 	.num_pipes = 3,
    402 	.has_ddi = 1,
    403 	.has_fpga_dbg = 1,
    404 	.has_fbc = 1,
    405 	GEN_DEFAULT_PIPEOFFSETS,
    406 	IVB_CURSOR_OFFSETS,
    407 };
    408 
    409 /*
    410  * Make sure any device matches here are from most specific to most
    411  * general.  For example, since the Quanta match is based on the subsystem
    412  * and subvendor IDs, we need it to come before the more general IVB
    413  * PCI ID matches, otherwise we'll use the wrong info struct above.
    414  */
    415 #define INTEL_PCI_IDS \
    416 	INTEL_I830_IDS(&intel_i830_info),	\
    417 	INTEL_I845G_IDS(&intel_845g_info),	\
    418 	INTEL_I85X_IDS(&intel_i85x_info),	\
    419 	INTEL_I865G_IDS(&intel_i865g_info),	\
    420 	INTEL_I915G_IDS(&intel_i915g_info),	\
    421 	INTEL_I915GM_IDS(&intel_i915gm_info),	\
    422 	INTEL_I945G_IDS(&intel_i945g_info),	\
    423 	INTEL_I945GM_IDS(&intel_i945gm_info),	\
    424 	INTEL_I965G_IDS(&intel_i965g_info),	\
    425 	INTEL_G33_IDS(&intel_g33_info),		\
    426 	INTEL_I965GM_IDS(&intel_i965gm_info),	\
    427 	INTEL_GM45_IDS(&intel_gm45_info), 	\
    428 	INTEL_G45_IDS(&intel_g45_info), 	\
    429 	INTEL_PINEVIEW_IDS(&intel_pineview_info),	\
    430 	INTEL_IRONLAKE_D_IDS(&intel_ironlake_d_info),	\
    431 	INTEL_IRONLAKE_M_IDS(&intel_ironlake_m_info),	\
    432 	INTEL_SNB_D_IDS(&intel_sandybridge_d_info),	\
    433 	INTEL_SNB_M_IDS(&intel_sandybridge_m_info),	\
    434 	INTEL_IVB_Q_IDS(&intel_ivybridge_q_info), /* must be first IVB */ \
    435 	INTEL_IVB_M_IDS(&intel_ivybridge_m_info),	\
    436 	INTEL_IVB_D_IDS(&intel_ivybridge_d_info),	\
    437 	INTEL_HSW_D_IDS(&intel_haswell_d_info), \
    438 	INTEL_HSW_M_IDS(&intel_haswell_m_info), \
    439 	INTEL_VLV_M_IDS(&intel_valleyview_m_info),	\
    440 	INTEL_VLV_D_IDS(&intel_valleyview_d_info),	\
    441 	INTEL_BDW_GT12M_IDS(&intel_broadwell_m_info),	\
    442 	INTEL_BDW_GT12D_IDS(&intel_broadwell_d_info),	\
    443 	INTEL_BDW_GT3M_IDS(&intel_broadwell_gt3m_info),	\
    444 	INTEL_BDW_GT3D_IDS(&intel_broadwell_gt3d_info), \
    445 	INTEL_CHV_IDS(&intel_cherryview_info),	\
    446 	INTEL_SKL_GT1_IDS(&intel_skylake_info),	\
    447 	INTEL_SKL_GT2_IDS(&intel_skylake_info),	\
    448 	INTEL_SKL_GT3_IDS(&intel_skylake_gt3_info),	\
    449 	INTEL_BXT_IDS(&intel_broxton_info)
    450 
    451 static const struct pci_device_id pciidlist[] = {		/* aka */
    452 	INTEL_PCI_IDS,
    453 	{0, 0, 0, 0, 0, 0, 0}
    454 };
    455 
    456 MODULE_DEVICE_TABLE(pci, pciidlist);
    457 
    458 static enum intel_pch intel_virt_detect_pch(struct drm_device *dev)
    459 {
    460 	enum intel_pch ret = PCH_NOP;
    461 
    462 	/*
    463 	 * In a virtualized passthrough environment we can be in a
    464 	 * setup where the ISA bridge is not able to be passed through.
    465 	 * In this case, a south bridge can be emulated and we have to
    466 	 * make an educated guess as to which PCH is really there.
    467 	 */
    468 
    469 	if (IS_GEN5(dev)) {
    470 		ret = PCH_IBX;
    471 		DRM_DEBUG_KMS("Assuming Ibex Peak PCH\n");
    472 	} else if (IS_GEN6(dev) || IS_IVYBRIDGE(dev)) {
    473 		ret = PCH_CPT;
    474 		DRM_DEBUG_KMS("Assuming CouarPoint PCH\n");
    475 	} else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
    476 		ret = PCH_LPT;
    477 		DRM_DEBUG_KMS("Assuming LynxPoint PCH\n");
    478 	} else if (IS_SKYLAKE(dev)) {
    479 		ret = PCH_SPT;
    480 		DRM_DEBUG_KMS("Assuming SunrisePoint PCH\n");
    481 	}
    482 
    483 	return ret;
    484 }
    485 
    486 #ifdef __NetBSD__
    487 /* XXX Kludge to expose this to NetBSD driver attachment goop.  */
    488 const struct pci_device_id *const i915_device_ids = pciidlist;
    489 const size_t i915_n_device_ids = __arraycount(pciidlist);
    490 #endif
    491 
    492 void intel_detect_pch(struct drm_device *dev)
    493 {
    494 	struct drm_i915_private *dev_priv = dev->dev_private;
    495 	struct pci_dev *pch = NULL;
    496 
    497 	/* In all current cases, num_pipes is equivalent to the PCH_NOP setting
    498 	 * (which really amounts to a PCH but no South Display).
    499 	 */
    500 	if (INTEL_INFO(dev)->num_pipes == 0) {
    501 		dev_priv->pch_type = PCH_NOP;
    502 		return;
    503 	}
    504 
    505 	/*
    506 	 * The reason to probe ISA bridge instead of Dev31:Fun0 is to
    507 	 * make graphics device passthrough work easy for VMM, that only
    508 	 * need to expose ISA bridge to let driver know the real hardware
    509 	 * underneath. This is a requirement from virtualization team.
    510 	 *
    511 	 * In some virtualized environments (e.g. XEN), there is irrelevant
    512 	 * ISA bridge in the system. To work reliably, we should scan trhough
    513 	 * all the ISA bridge devices and check for the first match, instead
    514 	 * of only checking the first one.
    515 	 */
    516 	while ((pch = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, pch))) {
    517 		if (pch->vendor == PCI_VENDOR_ID_INTEL) {
    518 			unsigned short id = pch->device & INTEL_PCH_DEVICE_ID_MASK;
    519 			dev_priv->pch_id = id;
    520 
    521 			if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) {
    522 				dev_priv->pch_type = PCH_IBX;
    523 				DRM_DEBUG_KMS("Found Ibex Peak PCH\n");
    524 				WARN_ON(!IS_GEN5(dev));
    525 			} else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) {
    526 				dev_priv->pch_type = PCH_CPT;
    527 				DRM_DEBUG_KMS("Found CougarPoint PCH\n");
    528 				WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
    529 			} else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) {
    530 				/* PantherPoint is CPT compatible */
    531 				dev_priv->pch_type = PCH_CPT;
    532 				DRM_DEBUG_KMS("Found PantherPoint PCH\n");
    533 				WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
    534 			} else if (id == INTEL_PCH_LPT_DEVICE_ID_TYPE) {
    535 				dev_priv->pch_type = PCH_LPT;
    536 				DRM_DEBUG_KMS("Found LynxPoint PCH\n");
    537 				WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
    538 				WARN_ON(IS_HSW_ULT(dev) || IS_BDW_ULT(dev));
    539 			} else if (id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
    540 				dev_priv->pch_type = PCH_LPT;
    541 				DRM_DEBUG_KMS("Found LynxPoint LP PCH\n");
    542 				WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
    543 				WARN_ON(!IS_HSW_ULT(dev) && !IS_BDW_ULT(dev));
    544 			} else if (id == INTEL_PCH_SPT_DEVICE_ID_TYPE) {
    545 				dev_priv->pch_type = PCH_SPT;
    546 				DRM_DEBUG_KMS("Found SunrisePoint PCH\n");
    547 				WARN_ON(!IS_SKYLAKE(dev));
    548 			} else if (id == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) {
    549 				dev_priv->pch_type = PCH_SPT;
    550 				DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n");
    551 				WARN_ON(!IS_SKYLAKE(dev));
    552 			} else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) ||
    553 				   ((id == INTEL_PCH_QEMU_DEVICE_ID_TYPE) &&
    554 				    pch->subsystem_vendor == 0x1af4 &&
    555 				    pch->subsystem_device == 0x1100)) {
    556 				dev_priv->pch_type = intel_virt_detect_pch(dev);
    557 			} else
    558 				continue;
    559 
    560 			break;
    561 		}
    562 	}
    563 	if (!pch)
    564 		DRM_DEBUG_KMS("No PCH found.\n");
    565 
    566 	pci_dev_put(pch);
    567 }
    568 
    569 bool i915_semaphore_is_enabled(struct drm_device *dev)
    570 {
    571 	if (INTEL_INFO(dev)->gen < 6)
    572 		return false;
    573 
    574 	if (i915.semaphores >= 0)
    575 		return i915.semaphores;
    576 
    577 	/* TODO: make semaphores and Execlists play nicely together */
    578 	if (i915.enable_execlists)
    579 		return false;
    580 
    581 	/* Until we get further testing... */
    582 	if (IS_GEN8(dev))
    583 		return false;
    584 
    585 #ifdef CONFIG_INTEL_IOMMU
    586 	/* Enable semaphores on SNB when IO remapping is off */
    587 	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
    588 		return false;
    589 #endif
    590 
    591 	return true;
    592 }
    593 
    594 void i915_firmware_load_error_print(const char *fw_path, int err)
    595 {
    596 	DRM_ERROR("failed to load firmware %s (%d)\n", fw_path, err);
    597 
    598 	/*
    599 	 * If the reason is not known assume -ENOENT since that's the most
    600 	 * usual failure mode.
    601 	 */
    602 	if (!err)
    603 		err = -ENOENT;
    604 
    605 	if (!(IS_BUILTIN(CONFIG_DRM_I915) && err == -ENOENT))
    606 		return;
    607 
    608 	DRM_ERROR(
    609 	  "The driver is built-in, so to load the firmware you need to\n"
    610 	  "include it either in the kernel (see CONFIG_EXTRA_FIRMWARE) or\n"
    611 	  "in your initrd/initramfs image.\n");
    612 }
    613 
    614 static void intel_suspend_encoders(struct drm_i915_private *dev_priv)
    615 {
    616 	struct drm_device *dev = dev_priv->dev;
    617 	struct drm_encoder *encoder;
    618 
    619 	drm_modeset_lock_all(dev);
    620 	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
    621 		struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
    622 
    623 		if (intel_encoder->suspend)
    624 			intel_encoder->suspend(intel_encoder);
    625 	}
    626 	drm_modeset_unlock_all(dev);
    627 }
    628 
    629 static int intel_suspend_complete(struct drm_i915_private *dev_priv);
    630 static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
    631 			      bool rpm_resume);
    632 static int skl_resume_prepare(struct drm_i915_private *dev_priv);
    633 static int bxt_resume_prepare(struct drm_i915_private *dev_priv);
    634 
    635 
    636 static int i915_drm_suspend(struct drm_device *dev)
    637 {
    638 	struct drm_i915_private *dev_priv = dev->dev_private;
    639 	pci_power_t opregion_target_state;
    640 	int error;
    641 
    642 	/* ignore lid events during suspend */
    643 	mutex_lock(&dev_priv->modeset_restore_lock);
    644 	dev_priv->modeset_restore = MODESET_SUSPENDED;
    645 	mutex_unlock(&dev_priv->modeset_restore_lock);
    646 
    647 	/* We do a lot of poking in a lot of registers, make sure they work
    648 	 * properly. */
    649 	intel_display_set_init_power(dev_priv, true);
    650 
    651 	drm_kms_helper_poll_disable(dev);
    652 
    653 #ifndef __NetBSD__		/* pmf handles this for us.  */
    654 	pci_save_state(dev->pdev);
    655 #endif
    656 
    657 	error = i915_gem_suspend(dev);
    658 	if (error) {
    659 #ifdef __NetBSD__
    660 		dev_err(pci_dev_dev(dev->pdev),
    661 		    "GEM idle failed, resume might fail\n");
    662 #else
    663 		dev_err(&dev->pdev->dev,
    664 			"GEM idle failed, resume might fail\n");
    665 #endif
    666 		return error;
    667 	}
    668 
    669 	intel_guc_suspend(dev);
    670 
    671 	intel_suspend_gt_powersave(dev);
    672 
    673 	/*
    674 	 * Disable CRTCs directly since we want to preserve sw state
    675 	 * for _thaw. Also, power gate the CRTC power wells.
    676 	 */
    677 	drm_modeset_lock_all(dev);
    678 	intel_display_suspend(dev);
    679 	drm_modeset_unlock_all(dev);
    680 
    681 	intel_dp_mst_suspend(dev);
    682 
    683 	intel_runtime_pm_disable_interrupts(dev_priv);
    684 	intel_hpd_cancel_work(dev_priv);
    685 
    686 	intel_suspend_encoders(dev_priv);
    687 
    688 	intel_suspend_hw(dev);
    689 
    690 	i915_gem_suspend_gtt_mappings(dev);
    691 
    692 	i915_save_state(dev);
    693 
    694 	opregion_target_state = PCI_D3cold;
    695 #if IS_ENABLED(CONFIG_ACPI_SLEEP)
    696 	if (acpi_target_system_state() < ACPI_STATE_S3)
    697 		opregion_target_state = PCI_D1;
    698 #endif
    699 	intel_opregion_notify_adapter(dev, opregion_target_state);
    700 
    701 	intel_uncore_forcewake_reset(dev, false);
    702 	intel_opregion_fini(dev);
    703 
    704 #ifndef __NetBSD__		/* XXX fb */
    705 	intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true);
    706 #endif
    707 
    708 	dev_priv->suspend_count++;
    709 
    710 	intel_display_set_init_power(dev_priv, false);
    711 
    712 	return 0;
    713 }
    714 
    715 static int i915_drm_suspend_late(struct drm_device *drm_dev, bool hibernation)
    716 {
    717 	struct drm_i915_private *dev_priv = drm_dev->dev_private;
    718 	int ret;
    719 
    720 	ret = intel_suspend_complete(dev_priv);
    721 
    722 	if (ret) {
    723 		DRM_ERROR("Suspend complete failed: %d\n", ret);
    724 
    725 		return ret;
    726 	}
    727 
    728 	pci_disable_device(drm_dev->pdev);
    729 	/*
    730 	 * During hibernation on some platforms the BIOS may try to access
    731 	 * the device even though it's already in D3 and hang the machine. So
    732 	 * leave the device in D0 on those platforms and hope the BIOS will
    733 	 * power down the device properly. The issue was seen on multiple old
    734 	 * GENs with different BIOS vendors, so having an explicit blacklist
    735 	 * is inpractical; apply the workaround on everything pre GEN6. The
    736 	 * platforms where the issue was seen:
    737 	 * Lenovo Thinkpad X301, X61s, X60, T60, X41
    738 	 * Fujitsu FSC S7110
    739 	 * Acer Aspire 1830T
    740 	 */
    741 	if (!(hibernation && INTEL_INFO(dev_priv)->gen < 6))
    742 		pci_set_power_state(drm_dev->pdev, PCI_D3hot);
    743 
    744 	return 0;
    745 }
    746 
    747 int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state)
    748 {
    749 	int error;
    750 
    751 	if (!dev || !dev->dev_private) {
    752 		DRM_ERROR("dev: %p\n", dev);
    753 		DRM_ERROR("DRM not initialized, aborting suspend.\n");
    754 		return -ENODEV;
    755 	}
    756 
    757 	if (WARN_ON_ONCE(state.event != PM_EVENT_SUSPEND &&
    758 			 state.event != PM_EVENT_FREEZE))
    759 		return -EINVAL;
    760 
    761 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
    762 		return 0;
    763 
    764 	error = i915_drm_suspend(dev);
    765 	if (error)
    766 		return error;
    767 
    768 	return i915_drm_suspend_late(dev, false);
    769 }
    770 
    771 static int i915_drm_resume(struct drm_device *dev)
    772 {
    773 	struct drm_i915_private *dev_priv = dev->dev_private;
    774 
    775 	mutex_lock(&dev->struct_mutex);
    776 	i915_gem_restore_gtt_mappings(dev);
    777 	mutex_unlock(&dev->struct_mutex);
    778 
    779 	i915_restore_state(dev);
    780 	intel_opregion_setup(dev);
    781 
    782 	intel_init_pch_refclk(dev);
    783 	drm_mode_config_reset(dev);
    784 
    785 	/*
    786 	 * Interrupts have to be enabled before any batches are run. If not the
    787 	 * GPU will hang. i915_gem_init_hw() will initiate batches to
    788 	 * update/restore the context.
    789 	 *
    790 	 * Modeset enabling in intel_modeset_init_hw() also needs working
    791 	 * interrupts.
    792 	 */
    793 	intel_runtime_pm_enable_interrupts(dev_priv);
    794 
    795 	mutex_lock(&dev->struct_mutex);
    796 	if (i915_gem_init_hw(dev)) {
    797 		DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
    798 			atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
    799 	}
    800 	mutex_unlock(&dev->struct_mutex);
    801 
    802 	intel_guc_resume(dev);
    803 
    804 	intel_modeset_init_hw(dev);
    805 
    806 	spin_lock_irq(&dev_priv->irq_lock);
    807 	if (dev_priv->display.hpd_irq_setup)
    808 		dev_priv->display.hpd_irq_setup(dev);
    809 	spin_unlock_irq(&dev_priv->irq_lock);
    810 
    811 	drm_modeset_lock_all(dev);
    812 	intel_display_resume(dev);
    813 	drm_modeset_unlock_all(dev);
    814 
    815 	intel_dp_mst_resume(dev);
    816 
    817 	/*
    818 	 * ... but also need to make sure that hotplug processing
    819 	 * doesn't cause havoc. Like in the driver load code we don't
    820 	 * bother with the tiny race here where we might loose hotplug
    821 	 * notifications.
    822 	 * */
    823 	intel_hpd_init(dev_priv);
    824 	/* Config may have changed between suspend and resume */
    825 	drm_helper_hpd_irq_event(dev);
    826 
    827 	intel_opregion_init(dev);
    828 
    829 #ifndef __NetBSD__		/* XXX fb */
    830 	intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false);
    831 #endif
    832 
    833 	mutex_lock(&dev_priv->modeset_restore_lock);
    834 	dev_priv->modeset_restore = MODESET_DONE;
    835 	mutex_unlock(&dev_priv->modeset_restore_lock);
    836 
    837 	intel_opregion_notify_adapter(dev, PCI_D0);
    838 
    839 	drm_kms_helper_poll_enable(dev);
    840 
    841 	return 0;
    842 }
    843 
    844 static int i915_drm_resume_early(struct drm_device *dev)
    845 {
    846 	struct drm_i915_private *dev_priv = dev->dev_private;
    847 	int ret = 0;
    848 
    849 #ifndef __NetBSD__		/* pmf handles this for us.  */
    850 	/*
    851 	 * We have a resume ordering issue with the snd-hda driver also
    852 	 * requiring our device to be power up. Due to the lack of a
    853 	 * parent/child relationship we currently solve this with an early
    854 	 * resume hook.
    855 	 *
    856 	 * FIXME: This should be solved with a special hdmi sink device or
    857 	 * similar so that power domains can be employed.
    858 	 */
    859 	if (pci_enable_device(dev->pdev))
    860 		return -EIO;
    861 #endif
    862 
    863 	/* XXX pmf probably handles this for us too.  */
    864 	pci_set_master(dev->pdev);
    865 
    866 	if (IS_VALLEYVIEW(dev_priv))
    867 		ret = vlv_resume_prepare(dev_priv, false);
    868 	if (ret)
    869 		DRM_ERROR("Resume prepare failed: %d, continuing anyway\n",
    870 			  ret);
    871 
    872 	intel_uncore_early_sanitize(dev, true);
    873 
    874 	if (IS_BROXTON(dev))
    875 		ret = bxt_resume_prepare(dev_priv);
    876 	else if (IS_SKYLAKE(dev_priv))
    877 		ret = skl_resume_prepare(dev_priv);
    878 	else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
    879 		hsw_disable_pc8(dev_priv);
    880 
    881 	intel_uncore_sanitize(dev);
    882 	intel_power_domains_init_hw(dev_priv);
    883 
    884 	return ret;
    885 }
    886 
    887 int i915_resume_switcheroo(struct drm_device *dev)
    888 {
    889 	int ret;
    890 
    891 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
    892 		return 0;
    893 
    894 	ret = i915_drm_resume_early(dev);
    895 	if (ret)
    896 		return ret;
    897 
    898 	return i915_drm_resume(dev);
    899 }
    900 
    901 /**
    902  * i915_reset - reset chip after a hang
    903  * @dev: drm device to reset
    904  *
    905  * Reset the chip.  Useful if a hang is detected. Returns zero on successful
    906  * reset or otherwise an error code.
    907  *
    908  * Procedure is fairly simple:
    909  *   - reset the chip using the reset reg
    910  *   - re-init context state
    911  *   - re-init hardware status page
    912  *   - re-init ring buffer
    913  *   - re-init interrupt state
    914  *   - re-init display
    915  */
    916 int i915_reset(struct drm_device *dev)
    917 {
    918 	struct drm_i915_private *dev_priv = dev->dev_private;
    919 	bool simulated;
    920 	int ret;
    921 
    922 	intel_reset_gt_powersave(dev);
    923 
    924 	mutex_lock(&dev->struct_mutex);
    925 
    926 	i915_gem_reset(dev);
    927 
    928 	simulated = dev_priv->gpu_error.stop_rings != 0;
    929 
    930 	ret = intel_gpu_reset(dev);
    931 
    932 	/* Also reset the gpu hangman. */
    933 	if (simulated) {
    934 		DRM_INFO("Simulated gpu hang, resetting stop_rings\n");
    935 		dev_priv->gpu_error.stop_rings = 0;
    936 		if (ret == -ENODEV) {
    937 			DRM_INFO("Reset not implemented, but ignoring "
    938 				 "error for simulated gpu hangs\n");
    939 			ret = 0;
    940 		}
    941 	}
    942 
    943 	if (i915_stop_ring_allow_warn(dev_priv))
    944 		pr_notice("drm/i915: Resetting chip after gpu hang\n");
    945 
    946 	if (ret) {
    947 		DRM_ERROR("Failed to reset chip: %i\n", ret);
    948 		mutex_unlock(&dev->struct_mutex);
    949 		return ret;
    950 	}
    951 
    952 	intel_overlay_reset(dev_priv);
    953 
    954 	/* Ok, now get things going again... */
    955 
    956 	/*
    957 	 * Everything depends on having the GTT running, so we need to start
    958 	 * there.  Fortunately we don't need to do this unless we reset the
    959 	 * chip at a PCI level.
    960 	 *
    961 	 * Next we need to restore the context, but we don't use those
    962 	 * yet either...
    963 	 *
    964 	 * Ring buffer needs to be re-initialized in the KMS case, or if X
    965 	 * was running at the time of the reset (i.e. we weren't VT
    966 	 * switched away).
    967 	 */
    968 
    969 	/* Used to prevent gem_check_wedged returning -EAGAIN during gpu reset */
    970 	dev_priv->gpu_error.reload_in_reset = true;
    971 
    972 	ret = i915_gem_init_hw(dev);
    973 
    974 	dev_priv->gpu_error.reload_in_reset = false;
    975 
    976 	mutex_unlock(&dev->struct_mutex);
    977 	if (ret) {
    978 		DRM_ERROR("Failed hw init on reset %d\n", ret);
    979 		return ret;
    980 	}
    981 
    982 	/*
    983 	 * rps/rc6 re-init is necessary to restore state lost after the
    984 	 * reset and the re-install of gt irqs. Skip for ironlake per
    985 	 * previous concerns that it doesn't respond well to some forms
    986 	 * of re-init after reset.
    987 	 */
    988 	if (INTEL_INFO(dev)->gen > 5)
    989 		intel_enable_gt_powersave(dev);
    990 
    991 	return 0;
    992 }
    993 
    994 #ifdef __NetBSD__
    995 
    996 static const struct uvm_pagerops i915_gem_uvm_ops = {
    997 	.pgo_reference = drm_gem_pager_reference,
    998 	.pgo_detach = drm_gem_pager_detach,
    999 	.pgo_fault = i915_gem_fault,
   1000 };
   1001 
   1002 #else
   1003 
   1004 static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
   1005 {
   1006 	struct intel_device_info *intel_info =
   1007 		(struct intel_device_info *) ent->driver_data;
   1008 
   1009 	if (IS_PRELIMINARY_HW(intel_info) && !i915.preliminary_hw_support) {
   1010 		DRM_INFO("This hardware requires preliminary hardware support.\n"
   1011 			 "See CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT, and/or modparam preliminary_hw_support\n");
   1012 		return -ENODEV;
   1013 	}
   1014 
   1015 	/* Only bind to function 0 of the device. Early generations
   1016 	 * used function 1 as a placeholder for multi-head. This causes
   1017 	 * us confusion instead, especially on the systems where both
   1018 	 * functions have the same PCI-ID!
   1019 	 */
   1020 	if (PCI_FUNC(pdev->devfn))
   1021 		return -ENODEV;
   1022 
   1023 	return drm_get_pci_dev(pdev, ent, &driver);
   1024 }
   1025 
   1026 static void
   1027 i915_pci_remove(struct pci_dev *pdev)
   1028 {
   1029 	struct drm_device *dev = pci_get_drvdata(pdev);
   1030 
   1031 	drm_put_dev(dev);
   1032 }
   1033 
   1034 #ifndef __NetBSD__
   1035 static int i915_pm_suspend(struct device *dev)
   1036 {
   1037 	struct pci_dev *pdev = to_pci_dev(dev);
   1038 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
   1039 
   1040 	if (!drm_dev || !drm_dev->dev_private) {
   1041 		dev_err(dev, "DRM not initialized, aborting suspend.\n");
   1042 		return -ENODEV;
   1043 	}
   1044 
   1045 	if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
   1046 		return 0;
   1047 
   1048 	return i915_drm_suspend(drm_dev);
   1049 }
   1050 
   1051 static int i915_pm_suspend_late(struct device *dev)
   1052 {
   1053 	struct drm_device *drm_dev = dev_to_i915(dev)->dev;
   1054 
   1055 	/*
   1056 	 * We have a suspend ordering issue with the snd-hda driver also
   1057 	 * requiring our device to be power up. Due to the lack of a
   1058 	 * parent/child relationship we currently solve this with an late
   1059 	 * suspend hook.
   1060 	 *
   1061 	 * FIXME: This should be solved with a special hdmi sink device or
   1062 	 * similar so that power domains can be employed.
   1063 	 */
   1064 	if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
   1065 		return 0;
   1066 
   1067 	return i915_drm_suspend_late(drm_dev, false);
   1068 }
   1069 
   1070 static int i915_pm_poweroff_late(struct device *dev)
   1071 {
   1072 	struct drm_device *drm_dev = dev_to_i915(dev)->dev;
   1073 
   1074 	if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
   1075 		return 0;
   1076 
   1077 	return i915_drm_suspend_late(drm_dev, true);
   1078 }
   1079 
   1080 static int i915_pm_resume_early(struct device *dev)
   1081 {
   1082 	struct drm_device *drm_dev = dev_to_i915(dev)->dev;
   1083 
   1084 	if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
   1085 		return 0;
   1086 
   1087 	return i915_drm_resume_early(drm_dev);
   1088 }
   1089 
   1090 static int i915_pm_resume(struct device *dev)
   1091 {
   1092 	struct drm_device *drm_dev = dev_to_i915(dev)->dev;
   1093 
   1094 	if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
   1095 		return 0;
   1096 
   1097 	return i915_drm_resume(drm_dev);
   1098 }
   1099 #endif
   1100 
   1101 static int skl_suspend_complete(struct drm_i915_private *dev_priv)
   1102 {
   1103 	/* Enabling DC6 is not a hard requirement to enter runtime D3 */
   1104 
   1105 	skl_uninit_cdclk(dev_priv);
   1106 
   1107 	return 0;
   1108 }
   1109 
   1110 static int hsw_suspend_complete(struct drm_i915_private *dev_priv)
   1111 {
   1112 	hsw_enable_pc8(dev_priv);
   1113 
   1114 	return 0;
   1115 }
   1116 
   1117 static int bxt_suspend_complete(struct drm_i915_private *dev_priv)
   1118 {
   1119 	struct drm_device *dev = dev_priv->dev;
   1120 
   1121 	/* TODO: when DC5 support is added disable DC5 here. */
   1122 
   1123 	broxton_ddi_phy_uninit(dev);
   1124 	broxton_uninit_cdclk(dev);
   1125 	bxt_enable_dc9(dev_priv);
   1126 
   1127 	return 0;
   1128 }
   1129 
   1130 static int bxt_resume_prepare(struct drm_i915_private *dev_priv)
   1131 {
   1132 	struct drm_device *dev = dev_priv->dev;
   1133 
   1134 	/* TODO: when CSR FW support is added make sure the FW is loaded */
   1135 
   1136 	bxt_disable_dc9(dev_priv);
   1137 
   1138 	/*
   1139 	 * TODO: when DC5 support is added enable DC5 here if the CSR FW
   1140 	 * is available.
   1141 	 */
   1142 	broxton_init_cdclk(dev);
   1143 	broxton_ddi_phy_init(dev);
   1144 	intel_prepare_ddi(dev);
   1145 
   1146 	return 0;
   1147 }
   1148 
   1149 static int skl_resume_prepare(struct drm_i915_private *dev_priv)
   1150 {
   1151 	struct drm_device *dev = dev_priv->dev;
   1152 
   1153 	skl_init_cdclk(dev_priv);
   1154 	intel_csr_load_program(dev);
   1155 
   1156 	return 0;
   1157 }
   1158 
   1159 /*
   1160  * Save all Gunit registers that may be lost after a D3 and a subsequent
   1161  * S0i[R123] transition. The list of registers needing a save/restore is
   1162  * defined in the VLV2_S0IXRegs document. This documents marks all Gunit
   1163  * registers in the following way:
   1164  * - Driver: saved/restored by the driver
   1165  * - Punit : saved/restored by the Punit firmware
   1166  * - No, w/o marking: no need to save/restore, since the register is R/O or
   1167  *                    used internally by the HW in a way that doesn't depend
   1168  *                    keeping the content across a suspend/resume.
   1169  * - Debug : used for debugging
   1170  *
   1171  * We save/restore all registers marked with 'Driver', with the following
   1172  * exceptions:
   1173  * - Registers out of use, including also registers marked with 'Debug'.
   1174  *   These have no effect on the driver's operation, so we don't save/restore
   1175  *   them to reduce the overhead.
   1176  * - Registers that are fully setup by an initialization function called from
   1177  *   the resume path. For example many clock gating and RPS/RC6 registers.
   1178  * - Registers that provide the right functionality with their reset defaults.
   1179  *
   1180  * TODO: Except for registers that based on the above 3 criteria can be safely
   1181  * ignored, we save/restore all others, practically treating the HW context as
   1182  * a black-box for the driver. Further investigation is needed to reduce the
   1183  * saved/restored registers even further, by following the same 3 criteria.
   1184  */
   1185 static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv)
   1186 {
   1187 	struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state;
   1188 	int i;
   1189 
   1190 	/* GAM 0x4000-0x4770 */
   1191 	s->wr_watermark		= I915_READ(GEN7_WR_WATERMARK);
   1192 	s->gfx_prio_ctrl	= I915_READ(GEN7_GFX_PRIO_CTRL);
   1193 	s->arb_mode		= I915_READ(ARB_MODE);
   1194 	s->gfx_pend_tlb0	= I915_READ(GEN7_GFX_PEND_TLB0);
   1195 	s->gfx_pend_tlb1	= I915_READ(GEN7_GFX_PEND_TLB1);
   1196 
   1197 	for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
   1198 		s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i));
   1199 
   1200 	s->media_max_req_count	= I915_READ(GEN7_MEDIA_MAX_REQ_COUNT);
   1201 	s->gfx_max_req_count	= I915_READ(GEN7_GFX_MAX_REQ_COUNT);
   1202 
   1203 	s->render_hwsp		= I915_READ(RENDER_HWS_PGA_GEN7);
   1204 	s->ecochk		= I915_READ(GAM_ECOCHK);
   1205 	s->bsd_hwsp		= I915_READ(BSD_HWS_PGA_GEN7);
   1206 	s->blt_hwsp		= I915_READ(BLT_HWS_PGA_GEN7);
   1207 
   1208 	s->tlb_rd_addr		= I915_READ(GEN7_TLB_RD_ADDR);
   1209 
   1210 	/* MBC 0x9024-0x91D0, 0x8500 */
   1211 	s->g3dctl		= I915_READ(VLV_G3DCTL);
   1212 	s->gsckgctl		= I915_READ(VLV_GSCKGCTL);
   1213 	s->mbctl		= I915_READ(GEN6_MBCTL);
   1214 
   1215 	/* GCP 0x9400-0x9424, 0x8100-0x810C */
   1216 	s->ucgctl1		= I915_READ(GEN6_UCGCTL1);
   1217 	s->ucgctl3		= I915_READ(GEN6_UCGCTL3);
   1218 	s->rcgctl1		= I915_READ(GEN6_RCGCTL1);
   1219 	s->rcgctl2		= I915_READ(GEN6_RCGCTL2);
   1220 	s->rstctl		= I915_READ(GEN6_RSTCTL);
   1221 	s->misccpctl		= I915_READ(GEN7_MISCCPCTL);
   1222 
   1223 	/* GPM 0xA000-0xAA84, 0x8000-0x80FC */
   1224 	s->gfxpause		= I915_READ(GEN6_GFXPAUSE);
   1225 	s->rpdeuhwtc		= I915_READ(GEN6_RPDEUHWTC);
   1226 	s->rpdeuc		= I915_READ(GEN6_RPDEUC);
   1227 	s->ecobus		= I915_READ(ECOBUS);
   1228 	s->pwrdwnupctl		= I915_READ(VLV_PWRDWNUPCTL);
   1229 	s->rp_down_timeout	= I915_READ(GEN6_RP_DOWN_TIMEOUT);
   1230 	s->rp_deucsw		= I915_READ(GEN6_RPDEUCSW);
   1231 	s->rcubmabdtmr		= I915_READ(GEN6_RCUBMABDTMR);
   1232 	s->rcedata		= I915_READ(VLV_RCEDATA);
   1233 	s->spare2gh		= I915_READ(VLV_SPAREG2H);
   1234 
   1235 	/* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
   1236 	s->gt_imr		= I915_READ(GTIMR);
   1237 	s->gt_ier		= I915_READ(GTIER);
   1238 	s->pm_imr		= I915_READ(GEN6_PMIMR);
   1239 	s->pm_ier		= I915_READ(GEN6_PMIER);
   1240 
   1241 	for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
   1242 		s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i));
   1243 
   1244 	/* GT SA CZ domain, 0x100000-0x138124 */
   1245 	s->tilectl		= I915_READ(TILECTL);
   1246 	s->gt_fifoctl		= I915_READ(GTFIFOCTL);
   1247 	s->gtlc_wake_ctrl	= I915_READ(VLV_GTLC_WAKE_CTRL);
   1248 	s->gtlc_survive		= I915_READ(VLV_GTLC_SURVIVABILITY_REG);
   1249 	s->pmwgicz		= I915_READ(VLV_PMWGICZ);
   1250 
   1251 	/* Gunit-Display CZ domain, 0x182028-0x1821CF */
   1252 	s->gu_ctl0		= I915_READ(VLV_GU_CTL0);
   1253 	s->gu_ctl1		= I915_READ(VLV_GU_CTL1);
   1254 	s->pcbr			= I915_READ(VLV_PCBR);
   1255 	s->clock_gate_dis2	= I915_READ(VLV_GUNIT_CLOCK_GATE2);
   1256 
   1257 	/*
   1258 	 * Not saving any of:
   1259 	 * DFT,		0x9800-0x9EC0
   1260 	 * SARB,	0xB000-0xB1FC
   1261 	 * GAC,		0x5208-0x524C, 0x14000-0x14C000
   1262 	 * PCI CFG
   1263 	 */
   1264 }
   1265 
   1266 static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv)
   1267 {
   1268 	struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state;
   1269 	u32 val;
   1270 	int i;
   1271 
   1272 	/* GAM 0x4000-0x4770 */
   1273 	I915_WRITE(GEN7_WR_WATERMARK,	s->wr_watermark);
   1274 	I915_WRITE(GEN7_GFX_PRIO_CTRL,	s->gfx_prio_ctrl);
   1275 	I915_WRITE(ARB_MODE,		s->arb_mode | (0xffff << 16));
   1276 	I915_WRITE(GEN7_GFX_PEND_TLB0,	s->gfx_pend_tlb0);
   1277 	I915_WRITE(GEN7_GFX_PEND_TLB1,	s->gfx_pend_tlb1);
   1278 
   1279 	for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
   1280 		I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]);
   1281 
   1282 	I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count);
   1283 	I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count);
   1284 
   1285 	I915_WRITE(RENDER_HWS_PGA_GEN7,	s->render_hwsp);
   1286 	I915_WRITE(GAM_ECOCHK,		s->ecochk);
   1287 	I915_WRITE(BSD_HWS_PGA_GEN7,	s->bsd_hwsp);
   1288 	I915_WRITE(BLT_HWS_PGA_GEN7,	s->blt_hwsp);
   1289 
   1290 	I915_WRITE(GEN7_TLB_RD_ADDR,	s->tlb_rd_addr);
   1291 
   1292 	/* MBC 0x9024-0x91D0, 0x8500 */
   1293 	I915_WRITE(VLV_G3DCTL,		s->g3dctl);
   1294 	I915_WRITE(VLV_GSCKGCTL,	s->gsckgctl);
   1295 	I915_WRITE(GEN6_MBCTL,		s->mbctl);
   1296 
   1297 	/* GCP 0x9400-0x9424, 0x8100-0x810C */
   1298 	I915_WRITE(GEN6_UCGCTL1,	s->ucgctl1);
   1299 	I915_WRITE(GEN6_UCGCTL3,	s->ucgctl3);
   1300 	I915_WRITE(GEN6_RCGCTL1,	s->rcgctl1);
   1301 	I915_WRITE(GEN6_RCGCTL2,	s->rcgctl2);
   1302 	I915_WRITE(GEN6_RSTCTL,		s->rstctl);
   1303 	I915_WRITE(GEN7_MISCCPCTL,	s->misccpctl);
   1304 
   1305 	/* GPM 0xA000-0xAA84, 0x8000-0x80FC */
   1306 	I915_WRITE(GEN6_GFXPAUSE,	s->gfxpause);
   1307 	I915_WRITE(GEN6_RPDEUHWTC,	s->rpdeuhwtc);
   1308 	I915_WRITE(GEN6_RPDEUC,		s->rpdeuc);
   1309 	I915_WRITE(ECOBUS,		s->ecobus);
   1310 	I915_WRITE(VLV_PWRDWNUPCTL,	s->pwrdwnupctl);
   1311 	I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout);
   1312 	I915_WRITE(GEN6_RPDEUCSW,	s->rp_deucsw);
   1313 	I915_WRITE(GEN6_RCUBMABDTMR,	s->rcubmabdtmr);
   1314 	I915_WRITE(VLV_RCEDATA,		s->rcedata);
   1315 	I915_WRITE(VLV_SPAREG2H,	s->spare2gh);
   1316 
   1317 	/* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
   1318 	I915_WRITE(GTIMR,		s->gt_imr);
   1319 	I915_WRITE(GTIER,		s->gt_ier);
   1320 	I915_WRITE(GEN6_PMIMR,		s->pm_imr);
   1321 	I915_WRITE(GEN6_PMIER,		s->pm_ier);
   1322 
   1323 	for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
   1324 		I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]);
   1325 
   1326 	/* GT SA CZ domain, 0x100000-0x138124 */
   1327 	I915_WRITE(TILECTL,			s->tilectl);
   1328 	I915_WRITE(GTFIFOCTL,			s->gt_fifoctl);
   1329 	/*
   1330 	 * Preserve the GT allow wake and GFX force clock bit, they are not
   1331 	 * be restored, as they are used to control the s0ix suspend/resume
   1332 	 * sequence by the caller.
   1333 	 */
   1334 	val = I915_READ(VLV_GTLC_WAKE_CTRL);
   1335 	val &= VLV_GTLC_ALLOWWAKEREQ;
   1336 	val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ;
   1337 	I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
   1338 
   1339 	val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
   1340 	val &= VLV_GFX_CLK_FORCE_ON_BIT;
   1341 	val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT;
   1342 	I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
   1343 
   1344 	I915_WRITE(VLV_PMWGICZ,			s->pmwgicz);
   1345 
   1346 	/* Gunit-Display CZ domain, 0x182028-0x1821CF */
   1347 	I915_WRITE(VLV_GU_CTL0,			s->gu_ctl0);
   1348 	I915_WRITE(VLV_GU_CTL1,			s->gu_ctl1);
   1349 	I915_WRITE(VLV_PCBR,			s->pcbr);
   1350 	I915_WRITE(VLV_GUNIT_CLOCK_GATE2,	s->clock_gate_dis2);
   1351 }
   1352 
   1353 int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on)
   1354 {
   1355 	u32 val;
   1356 	int err;
   1357 
   1358 #define COND (I915_READ(VLV_GTLC_SURVIVABILITY_REG) & VLV_GFX_CLK_STATUS_BIT)
   1359 
   1360 	val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
   1361 	val &= ~VLV_GFX_CLK_FORCE_ON_BIT;
   1362 	if (force_on)
   1363 		val |= VLV_GFX_CLK_FORCE_ON_BIT;
   1364 	I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
   1365 
   1366 	if (!force_on)
   1367 		return 0;
   1368 
   1369 	err = wait_for(COND, 20);
   1370 	if (err)
   1371 		DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n",
   1372 			  I915_READ(VLV_GTLC_SURVIVABILITY_REG));
   1373 
   1374 	return err;
   1375 #undef COND
   1376 }
   1377 
   1378 static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow)
   1379 {
   1380 	u32 val;
   1381 	int err = 0;
   1382 
   1383 	val = I915_READ(VLV_GTLC_WAKE_CTRL);
   1384 	val &= ~VLV_GTLC_ALLOWWAKEREQ;
   1385 	if (allow)
   1386 		val |= VLV_GTLC_ALLOWWAKEREQ;
   1387 	I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
   1388 	POSTING_READ(VLV_GTLC_WAKE_CTRL);
   1389 
   1390 #define COND (!!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEACK) == \
   1391 	      allow)
   1392 	err = wait_for(COND, 1);
   1393 	if (err)
   1394 		DRM_ERROR("timeout disabling GT waking\n");
   1395 	return err;
   1396 #undef COND
   1397 }
   1398 
   1399 static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
   1400 				 bool wait_for_on)
   1401 {
   1402 	u32 mask;
   1403 	u32 val;
   1404 	int err;
   1405 
   1406 	mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;
   1407 	val = wait_for_on ? mask : 0;
   1408 #define COND ((I915_READ(VLV_GTLC_PW_STATUS) & mask) == val)
   1409 	if (COND)
   1410 		return 0;
   1411 
   1412 	DRM_DEBUG_KMS("waiting for GT wells to go %s (%08x)\n",
   1413 			wait_for_on ? "on" : "off",
   1414 			I915_READ(VLV_GTLC_PW_STATUS));
   1415 
   1416 	/*
   1417 	 * RC6 transitioning can be delayed up to 2 msec (see
   1418 	 * valleyview_enable_rps), use 3 msec for safety.
   1419 	 */
   1420 	err = wait_for(COND, 3);
   1421 	if (err)
   1422 		DRM_ERROR("timeout waiting for GT wells to go %s\n",
   1423 			  wait_for_on ? "on" : "off");
   1424 
   1425 	return err;
   1426 #undef COND
   1427 }
   1428 
   1429 static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv)
   1430 {
   1431 	if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR))
   1432 		return;
   1433 
   1434 	DRM_ERROR("GT register access while GT waking disabled\n");
   1435 	I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
   1436 }
   1437 
   1438 static int vlv_suspend_complete(struct drm_i915_private *dev_priv)
   1439 {
   1440 	u32 mask;
   1441 	int err;
   1442 
   1443 	/*
   1444 	 * Bspec defines the following GT well on flags as debug only, so
   1445 	 * don't treat them as hard failures.
   1446 	 */
   1447 	(void)vlv_wait_for_gt_wells(dev_priv, false);
   1448 
   1449 	mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS;
   1450 	WARN_ON((I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask);
   1451 
   1452 	vlv_check_no_gt_access(dev_priv);
   1453 
   1454 	err = vlv_force_gfx_clock(dev_priv, true);
   1455 	if (err)
   1456 		goto err1;
   1457 
   1458 	err = vlv_allow_gt_wake(dev_priv, false);
   1459 	if (err)
   1460 		goto err2;
   1461 
   1462 	if (!IS_CHERRYVIEW(dev_priv->dev))
   1463 		vlv_save_gunit_s0ix_state(dev_priv);
   1464 
   1465 	err = vlv_force_gfx_clock(dev_priv, false);
   1466 	if (err)
   1467 		goto err2;
   1468 
   1469 	return 0;
   1470 
   1471 err2:
   1472 	/* For safety always re-enable waking and disable gfx clock forcing */
   1473 	vlv_allow_gt_wake(dev_priv, true);
   1474 err1:
   1475 	vlv_force_gfx_clock(dev_priv, false);
   1476 
   1477 	return err;
   1478 }
   1479 
   1480 static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
   1481 				bool rpm_resume)
   1482 {
   1483 	struct drm_device *dev = dev_priv->dev;
   1484 	int err;
   1485 	int ret;
   1486 
   1487 	/*
   1488 	 * If any of the steps fail just try to continue, that's the best we
   1489 	 * can do at this point. Return the first error code (which will also
   1490 	 * leave RPM permanently disabled).
   1491 	 */
   1492 	ret = vlv_force_gfx_clock(dev_priv, true);
   1493 
   1494 	if (!IS_CHERRYVIEW(dev_priv->dev))
   1495 		vlv_restore_gunit_s0ix_state(dev_priv);
   1496 
   1497 	err = vlv_allow_gt_wake(dev_priv, true);
   1498 	if (!ret)
   1499 		ret = err;
   1500 
   1501 	err = vlv_force_gfx_clock(dev_priv, false);
   1502 	if (!ret)
   1503 		ret = err;
   1504 
   1505 	vlv_check_no_gt_access(dev_priv);
   1506 
   1507 	if (rpm_resume) {
   1508 		intel_init_clock_gating(dev);
   1509 		i915_gem_restore_fences(dev);
   1510 	}
   1511 
   1512 	return ret;
   1513 }
   1514 
   1515 static int intel_runtime_suspend(struct device *device)
   1516 {
   1517 	struct pci_dev *pdev = to_pci_dev(device);
   1518 	struct drm_device *dev = pci_get_drvdata(pdev);
   1519 	struct drm_i915_private *dev_priv = dev->dev_private;
   1520 	int ret;
   1521 
   1522 	if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6(dev))))
   1523 		return -ENODEV;
   1524 
   1525 	if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
   1526 		return -ENODEV;
   1527 
   1528 	DRM_DEBUG_KMS("Suspending device\n");
   1529 
   1530 	/*
   1531 	 * We could deadlock here in case another thread holding struct_mutex
   1532 	 * calls RPM suspend concurrently, since the RPM suspend will wait
   1533 	 * first for this RPM suspend to finish. In this case the concurrent
   1534 	 * RPM resume will be followed by its RPM suspend counterpart. Still
   1535 	 * for consistency return -EAGAIN, which will reschedule this suspend.
   1536 	 */
   1537 	if (!mutex_trylock(&dev->struct_mutex)) {
   1538 		DRM_DEBUG_KMS("device lock contention, deffering suspend\n");
   1539 		/*
   1540 		 * Bump the expiration timestamp, otherwise the suspend won't
   1541 		 * be rescheduled.
   1542 		 */
   1543 		pm_runtime_mark_last_busy(device);
   1544 
   1545 		return -EAGAIN;
   1546 	}
   1547 	/*
   1548 	 * We are safe here against re-faults, since the fault handler takes
   1549 	 * an RPM reference.
   1550 	 */
   1551 	i915_gem_release_all_mmaps(dev_priv);
   1552 	mutex_unlock(&dev->struct_mutex);
   1553 
   1554 	intel_guc_suspend(dev);
   1555 
   1556 	intel_suspend_gt_powersave(dev);
   1557 	intel_runtime_pm_disable_interrupts(dev_priv);
   1558 
   1559 	ret = intel_suspend_complete(dev_priv);
   1560 	if (ret) {
   1561 		DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret);
   1562 		intel_runtime_pm_enable_interrupts(dev_priv);
   1563 
   1564 		return ret;
   1565 	}
   1566 
   1567 	cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
   1568 	intel_uncore_forcewake_reset(dev, false);
   1569 	dev_priv->pm.suspended = true;
   1570 
   1571 	/*
   1572 	 * FIXME: We really should find a document that references the arguments
   1573 	 * used below!
   1574 	 */
   1575 	if (IS_BROADWELL(dev)) {
   1576 		/*
   1577 		 * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
   1578 		 * being detected, and the call we do at intel_runtime_resume()
   1579 		 * won't be able to restore them. Since PCI_D3hot matches the
   1580 		 * actual specification and appears to be working, use it.
   1581 		 */
   1582 		intel_opregion_notify_adapter(dev, PCI_D3hot);
   1583 	} else {
   1584 		/*
   1585 		 * current versions of firmware which depend on this opregion
   1586 		 * notification have repurposed the D1 definition to mean
   1587 		 * "runtime suspended" vs. what you would normally expect (D3)
   1588 		 * to distinguish it from notifications that might be sent via
   1589 		 * the suspend path.
   1590 		 */
   1591 		intel_opregion_notify_adapter(dev, PCI_D1);
   1592 	}
   1593 
   1594 	assert_forcewakes_inactive(dev_priv);
   1595 
   1596 	DRM_DEBUG_KMS("Device suspended\n");
   1597 	return 0;
   1598 }
   1599 
   1600 static int intel_runtime_resume(struct device *device)
   1601 {
   1602 	struct pci_dev *pdev = to_pci_dev(device);
   1603 	struct drm_device *dev = pci_get_drvdata(pdev);
   1604 	struct drm_i915_private *dev_priv = dev->dev_private;
   1605 	int ret = 0;
   1606 
   1607 	if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
   1608 		return -ENODEV;
   1609 
   1610 	DRM_DEBUG_KMS("Resuming device\n");
   1611 
   1612 	intel_opregion_notify_adapter(dev, PCI_D0);
   1613 	dev_priv->pm.suspended = false;
   1614 
   1615 	intel_guc_resume(dev);
   1616 
   1617 	if (IS_GEN6(dev_priv))
   1618 		intel_init_pch_refclk(dev);
   1619 
   1620 	if (IS_BROXTON(dev))
   1621 		ret = bxt_resume_prepare(dev_priv);
   1622 	else if (IS_SKYLAKE(dev))
   1623 		ret = skl_resume_prepare(dev_priv);
   1624 	else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
   1625 		hsw_disable_pc8(dev_priv);
   1626 	else if (IS_VALLEYVIEW(dev_priv))
   1627 		ret = vlv_resume_prepare(dev_priv, true);
   1628 
   1629 	/*
   1630 	 * No point of rolling back things in case of an error, as the best
   1631 	 * we can do is to hope that things will still work (and disable RPM).
   1632 	 */
   1633 	i915_gem_init_swizzling(dev);
   1634 	gen6_update_ring_freq(dev);
   1635 
   1636 	intel_runtime_pm_enable_interrupts(dev_priv);
   1637 
   1638 	/*
   1639 	 * On VLV/CHV display interrupts are part of the display
   1640 	 * power well, so hpd is reinitialized from there. For
   1641 	 * everyone else do it here.
   1642 	 */
   1643 	if (!IS_VALLEYVIEW(dev_priv))
   1644 		intel_hpd_init(dev_priv);
   1645 
   1646 	intel_enable_gt_powersave(dev);
   1647 
   1648 	if (ret)
   1649 		DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret);
   1650 	else
   1651 		DRM_DEBUG_KMS("Device resumed\n");
   1652 
   1653 	return ret;
   1654 }
   1655 
   1656 /*
   1657  * This function implements common functionality of runtime and system
   1658  * suspend sequence.
   1659  */
   1660 static int intel_suspend_complete(struct drm_i915_private *dev_priv)
   1661 {
   1662 	int ret;
   1663 
   1664 	if (IS_BROXTON(dev_priv))
   1665 		ret = bxt_suspend_complete(dev_priv);
   1666 	else if (IS_SKYLAKE(dev_priv))
   1667 		ret = skl_suspend_complete(dev_priv);
   1668 	else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
   1669 		ret = hsw_suspend_complete(dev_priv);
   1670 	else if (IS_VALLEYVIEW(dev_priv))
   1671 		ret = vlv_suspend_complete(dev_priv);
   1672 	else
   1673 		ret = 0;
   1674 
   1675 	return ret;
   1676 }
   1677 
   1678 static const struct dev_pm_ops i915_pm_ops = {
   1679 	/*
   1680 	 * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND,
   1681 	 * PMSG_RESUME]
   1682 	 */
   1683 	.suspend = i915_pm_suspend,
   1684 	.suspend_late = i915_pm_suspend_late,
   1685 	.resume_early = i915_pm_resume_early,
   1686 	.resume = i915_pm_resume,
   1687 
   1688 	/*
   1689 	 * S4 event handlers
   1690 	 * @freeze, @freeze_late    : called (1) before creating the
   1691 	 *                            hibernation image [PMSG_FREEZE] and
   1692 	 *                            (2) after rebooting, before restoring
   1693 	 *                            the image [PMSG_QUIESCE]
   1694 	 * @thaw, @thaw_early       : called (1) after creating the hibernation
   1695 	 *                            image, before writing it [PMSG_THAW]
   1696 	 *                            and (2) after failing to create or
   1697 	 *                            restore the image [PMSG_RECOVER]
   1698 	 * @poweroff, @poweroff_late: called after writing the hibernation
   1699 	 *                            image, before rebooting [PMSG_HIBERNATE]
   1700 	 * @restore, @restore_early : called after rebooting and restoring the
   1701 	 *                            hibernation image [PMSG_RESTORE]
   1702 	 */
   1703 	.freeze = i915_pm_suspend,
   1704 	.freeze_late = i915_pm_suspend_late,
   1705 	.thaw_early = i915_pm_resume_early,
   1706 	.thaw = i915_pm_resume,
   1707 	.poweroff = i915_pm_suspend,
   1708 	.poweroff_late = i915_pm_poweroff_late,
   1709 	.restore_early = i915_pm_resume_early,
   1710 	.restore = i915_pm_resume,
   1711 
   1712 	/* S0ix (via runtime suspend) event handlers */
   1713 	.runtime_suspend = intel_runtime_suspend,
   1714 	.runtime_resume = intel_runtime_resume,
   1715 };
   1716 
   1717 static const struct vm_operations_struct i915_gem_vm_ops = {
   1718 	.fault = i915_gem_fault,
   1719 	.open = drm_gem_vm_open,
   1720 	.close = drm_gem_vm_close,
   1721 };
   1722 
   1723 static const struct file_operations i915_driver_fops = {
   1724 	.owner = THIS_MODULE,
   1725 	.open = drm_open,
   1726 	.release = drm_release,
   1727 	.unlocked_ioctl = drm_ioctl,
   1728 	.mmap = drm_gem_mmap,
   1729 	.poll = drm_poll,
   1730 	.read = drm_read,
   1731 #ifdef CONFIG_COMPAT
   1732 	.compat_ioctl = i915_compat_ioctl,
   1733 #endif
   1734 	.llseek = noop_llseek,
   1735 };
   1736 
   1737 #endif	/* defined(__NetBSD__) */
   1738 
   1739 static struct drm_driver driver = {
   1740 	/* Don't use MTRRs here; the Xserver or userspace app should
   1741 	 * deal with them for Intel hardware.
   1742 	 */
   1743 	.driver_features =
   1744 	    DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | DRIVER_PRIME |
   1745 	    DRIVER_RENDER | DRIVER_MODESET,
   1746 	.load = i915_driver_load,
   1747 	.unload = i915_driver_unload,
   1748 	.open = i915_driver_open,
   1749 	.lastclose = i915_driver_lastclose,
   1750 	.preclose = i915_driver_preclose,
   1751 	.postclose = i915_driver_postclose,
   1752 	.set_busid = drm_pci_set_busid,
   1753 #ifdef __NetBSD__
   1754 	.request_irq = drm_pci_request_irq,
   1755 	.free_irq = drm_pci_free_irq,
   1756 #endif
   1757 
   1758 #if defined(CONFIG_DEBUG_FS)
   1759 	.debugfs_init = i915_debugfs_init,
   1760 	.debugfs_cleanup = i915_debugfs_cleanup,
   1761 #endif
   1762 	.gem_free_object = i915_gem_free_object,
   1763 #ifdef __NetBSD__
   1764 	/* XXX Not clear the `or legacy' part is important here.  */
   1765 	.mmap_object = &drm_gem_or_legacy_mmap_object,
   1766 	.gem_uvm_ops = &i915_gem_uvm_ops,
   1767 #else
   1768 	.gem_vm_ops = &i915_gem_vm_ops,
   1769 #endif
   1770 
   1771 #ifndef __NetBSD__		/* XXX drm prime */
   1772 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
   1773 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
   1774 	.gem_prime_export = i915_gem_prime_export,
   1775 	.gem_prime_import = i915_gem_prime_import,
   1776 #endif
   1777 
   1778 	.dumb_create = i915_gem_dumb_create,
   1779 	.dumb_map_offset = i915_gem_mmap_gtt,
   1780 	.dumb_destroy = drm_gem_dumb_destroy,
   1781 	.ioctls = i915_ioctls,
   1782 #ifdef __NetBSD__
   1783 	.fops = NULL,
   1784 #else
   1785 	.fops = &i915_driver_fops,
   1786 #endif
   1787 	.name = DRIVER_NAME,
   1788 	.desc = DRIVER_DESC,
   1789 	.date = DRIVER_DATE,
   1790 	.major = DRIVER_MAJOR,
   1791 	.minor = DRIVER_MINOR,
   1792 	.patchlevel = DRIVER_PATCHLEVEL,
   1793 };
   1794 
   1795 #ifndef __NetBSD__
   1796 static struct pci_driver i915_pci_driver = {
   1797 	.name = DRIVER_NAME,
   1798 	.id_table = pciidlist,
   1799 	.probe = i915_pci_probe,
   1800 	.remove = i915_pci_remove,
   1801 	.driver.pm = &i915_pm_ops,
   1802 };
   1803 #endif
   1804 
   1805 #ifndef __NetBSD__
   1806 static int __init i915_init(void)
   1807 {
   1808 	driver.num_ioctls = i915_max_ioctl;
   1809 
   1810 	/*
   1811 	 * Enable KMS by default, unless explicitly overriden by
   1812 	 * either the i915.modeset prarameter or by the
   1813 	 * vga_text_mode_force boot option.
   1814 	 */
   1815 
   1816 	if (i915.modeset == 0)
   1817 		driver.driver_features &= ~DRIVER_MODESET;
   1818 
   1819 #ifdef CONFIG_VGA_CONSOLE
   1820 	if (vgacon_text_force() && i915.modeset == -1)
   1821 		driver.driver_features &= ~DRIVER_MODESET;
   1822 #endif
   1823 
   1824 	if (!(driver.driver_features & DRIVER_MODESET)) {
   1825 		/* Silently fail loading to not upset userspace. */
   1826 		DRM_DEBUG_DRIVER("KMS and UMS disabled.\n");
   1827 		return 0;
   1828 	}
   1829 
   1830 	if (i915.nuclear_pageflip)
   1831 		driver.driver_features |= DRIVER_ATOMIC;
   1832 
   1833 	return drm_pci_init(&driver, &i915_pci_driver);
   1834 }
   1835 
   1836 static void __exit i915_exit(void)
   1837 {
   1838 	if (!(driver.driver_features & DRIVER_MODESET))
   1839 		return; /* Never loaded a driver. */
   1840 
   1841 	drm_pci_exit(&driver, &i915_pci_driver);
   1842 }
   1843 
   1844 module_init(i915_init);
   1845 module_exit(i915_exit);
   1846 #endif
   1847 
   1848 MODULE_AUTHOR("Tungsten Graphics, Inc.");
   1849 MODULE_AUTHOR("Intel Corporation");
   1850 
   1851 MODULE_DESCRIPTION(DRIVER_DESC);
   1852 MODULE_LICENSE("GPL and additional rights");
   1853