intel_pm.c revision 1.1.1.4
1 /* $NetBSD: intel_pm.c,v 1.1.1.4 2021/12/18 20:15:26 riastradh Exp $ */ 2 3 /* 4 * Copyright 2012 Intel Corporation 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 * IN THE SOFTWARE. 24 * 25 * Authors: 26 * Eugeni Dodonov <eugeni.dodonov (at) intel.com> 27 * 28 */ 29 30 #include <sys/cdefs.h> 31 __KERNEL_RCSID(0, "$NetBSD: intel_pm.c,v 1.1.1.4 2021/12/18 20:15:26 riastradh Exp $"); 32 33 #include <linux/module.h> 34 #include <linux/pm_runtime.h> 35 36 #include <drm/drm_atomic_helper.h> 37 #include <drm/drm_fourcc.h> 38 #include <drm/drm_plane_helper.h> 39 40 #include "display/intel_atomic.h" 41 #include "display/intel_display_types.h" 42 #include "display/intel_fbc.h" 43 #include "display/intel_sprite.h" 44 45 #include "gt/intel_llc.h" 46 47 #include "i915_drv.h" 48 #include "i915_irq.h" 49 #include "i915_trace.h" 50 #include "intel_pm.h" 51 #include "intel_sideband.h" 52 #include "../../../platform/x86/intel_ips.h" 53 54 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv) 55 { 56 if (HAS_LLC(dev_priv)) { 57 /* 58 * WaCompressedResourceDisplayNewHashMode:skl,kbl 59 * Display WA #0390: skl,kbl 60 * 61 * Must match Sampler, Pixel Back End, and Media. See 62 * WaCompressedResourceSamplerPbeMediaNewHashMode. 63 */ 64 I915_WRITE(CHICKEN_PAR1_1, 65 I915_READ(CHICKEN_PAR1_1) | 66 SKL_DE_COMPRESSED_HASH_MODE); 67 } 68 69 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */ 70 I915_WRITE(CHICKEN_PAR1_1, 71 I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP); 72 73 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */ 74 I915_WRITE(GEN8_CHICKEN_DCPR_1, 75 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM); 76 77 /* WaFbcTurnOffFbcWatermark:skl,bxt,kbl,cfl */ 78 /* WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl */ 79 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 80 DISP_FBC_WM_DIS | 81 DISP_FBC_MEMORY_WAKE); 82 83 /* WaFbcHighMemBwCorruptionAvoidance:skl,bxt,kbl,cfl */ 84 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | 85 ILK_DPFC_DISABLE_DUMMY0); 86 87 if (IS_SKYLAKE(dev_priv)) { 88 /* WaDisableDopClockGating */ 89 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL) 90 & ~GEN7_DOP_CLOCK_GATE_ENABLE); 91 } 92 } 93 94 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv) 95 { 96 gen9_init_clock_gating(dev_priv); 97 98 /* WaDisableSDEUnitClockGating:bxt */ 99 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | 100 GEN8_SDEUNIT_CLOCK_GATE_DISABLE); 101 102 /* 103 * FIXME: 104 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only. 105 */ 106 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | 107 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ); 108 109 /* 110 * Wa: Backlight PWM may stop in the asserted state, causing backlight 111 * to stay fully on. 112 */ 113 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) | 114 PWM1_GATING_DIS | PWM2_GATING_DIS); 115 116 /* 117 * Lower the display internal timeout. 118 * This is needed to avoid any hard hangs when DSI port PLL 119 * is off and a MMIO access is attempted by any privilege 120 * application, using batch buffers or any other means. 121 */ 122 I915_WRITE(RM_TIMEOUT, MMIO_TIMEOUT_US(950)); 123 } 124 125 static void glk_init_clock_gating(struct drm_i915_private *dev_priv) 126 { 127 gen9_init_clock_gating(dev_priv); 128 129 /* 130 * WaDisablePWMClockGating:glk 131 * Backlight PWM may stop in the asserted state, causing backlight 132 * to stay fully on. 133 */ 134 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) | 135 PWM1_GATING_DIS | PWM2_GATING_DIS); 136 137 /* WaDDIIOTimeout:glk */ 138 if (IS_GLK_REVID(dev_priv, 0, GLK_REVID_A1)) { 139 u32 val = I915_READ(CHICKEN_MISC_2); 140 val &= ~(GLK_CL0_PWR_DOWN | 141 GLK_CL1_PWR_DOWN | 142 GLK_CL2_PWR_DOWN); 143 I915_WRITE(CHICKEN_MISC_2, val); 144 } 145 146 } 147 148 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv) 149 { 150 u32 tmp; 151 152 tmp = I915_READ(CLKCFG); 153 154 switch (tmp & CLKCFG_FSB_MASK) { 155 case CLKCFG_FSB_533: 156 dev_priv->fsb_freq = 533; /* 133*4 */ 157 break; 158 case CLKCFG_FSB_800: 159 dev_priv->fsb_freq = 800; /* 200*4 */ 160 break; 161 case CLKCFG_FSB_667: 162 dev_priv->fsb_freq = 667; /* 167*4 */ 163 break; 164 case CLKCFG_FSB_400: 165 dev_priv->fsb_freq = 400; /* 100*4 */ 166 break; 167 } 168 169 switch (tmp & CLKCFG_MEM_MASK) { 170 case CLKCFG_MEM_533: 171 dev_priv->mem_freq = 533; 172 break; 173 case CLKCFG_MEM_667: 174 dev_priv->mem_freq = 667; 175 break; 176 case CLKCFG_MEM_800: 177 dev_priv->mem_freq = 800; 178 break; 179 } 180 181 /* detect pineview DDR3 setting */ 182 tmp = I915_READ(CSHRDDR3CTL); 183 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0; 184 } 185 186 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv) 187 { 188 u16 ddrpll, csipll; 189 190 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1); 191 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0); 192 193 switch (ddrpll & 0xff) { 194 case 0xc: 195 dev_priv->mem_freq = 800; 196 break; 197 case 0x10: 198 dev_priv->mem_freq = 1066; 199 break; 200 case 0x14: 201 dev_priv->mem_freq = 1333; 202 break; 203 case 0x18: 204 dev_priv->mem_freq = 1600; 205 break; 206 default: 207 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n", 208 ddrpll & 0xff); 209 dev_priv->mem_freq = 0; 210 break; 211 } 212 213 switch (csipll & 0x3ff) { 214 case 0x00c: 215 dev_priv->fsb_freq = 3200; 216 break; 217 case 0x00e: 218 dev_priv->fsb_freq = 3733; 219 break; 220 case 0x010: 221 dev_priv->fsb_freq = 4266; 222 break; 223 case 0x012: 224 dev_priv->fsb_freq = 4800; 225 break; 226 case 0x014: 227 dev_priv->fsb_freq = 5333; 228 break; 229 case 0x016: 230 dev_priv->fsb_freq = 5866; 231 break; 232 case 0x018: 233 dev_priv->fsb_freq = 6400; 234 break; 235 default: 236 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n", 237 csipll & 0x3ff); 238 dev_priv->fsb_freq = 0; 239 break; 240 } 241 } 242 243 static const struct cxsr_latency cxsr_latency_table[] = { 244 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */ 245 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */ 246 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */ 247 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */ 248 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */ 249 250 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */ 251 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */ 252 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */ 253 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */ 254 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */ 255 256 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */ 257 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */ 258 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */ 259 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */ 260 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */ 261 262 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */ 263 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */ 264 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */ 265 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */ 266 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */ 267 268 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */ 269 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */ 270 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */ 271 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */ 272 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */ 273 274 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */ 275 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */ 276 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */ 277 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */ 278 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */ 279 }; 280 281 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop, 282 bool is_ddr3, 283 int fsb, 284 int mem) 285 { 286 const struct cxsr_latency *latency; 287 int i; 288 289 if (fsb == 0 || mem == 0) 290 return NULL; 291 292 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) { 293 latency = &cxsr_latency_table[i]; 294 if (is_desktop == latency->is_desktop && 295 is_ddr3 == latency->is_ddr3 && 296 fsb == latency->fsb_freq && mem == latency->mem_freq) 297 return latency; 298 } 299 300 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n"); 301 302 return NULL; 303 } 304 305 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable) 306 { 307 u32 val; 308 309 vlv_punit_get(dev_priv); 310 311 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2); 312 if (enable) 313 val &= ~FORCE_DDR_HIGH_FREQ; 314 else 315 val |= FORCE_DDR_HIGH_FREQ; 316 val &= ~FORCE_DDR_LOW_FREQ; 317 val |= FORCE_DDR_FREQ_REQ_ACK; 318 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val); 319 320 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) & 321 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) 322 drm_err(&dev_priv->drm, 323 "timed out waiting for Punit DDR DVFS request\n"); 324 325 vlv_punit_put(dev_priv); 326 } 327 328 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable) 329 { 330 u32 val; 331 332 vlv_punit_get(dev_priv); 333 334 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM); 335 if (enable) 336 val |= DSP_MAXFIFO_PM5_ENABLE; 337 else 338 val &= ~DSP_MAXFIFO_PM5_ENABLE; 339 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val); 340 341 vlv_punit_put(dev_priv); 342 } 343 344 #define FW_WM(value, plane) \ 345 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK) 346 347 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) 348 { 349 bool was_enabled; 350 u32 val; 351 352 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { 353 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN; 354 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0); 355 POSTING_READ(FW_BLC_SELF_VLV); 356 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) { 357 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN; 358 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0); 359 POSTING_READ(FW_BLC_SELF); 360 } else if (IS_PINEVIEW(dev_priv)) { 361 val = I915_READ(DSPFW3); 362 was_enabled = val & PINEVIEW_SELF_REFRESH_EN; 363 if (enable) 364 val |= PINEVIEW_SELF_REFRESH_EN; 365 else 366 val &= ~PINEVIEW_SELF_REFRESH_EN; 367 I915_WRITE(DSPFW3, val); 368 POSTING_READ(DSPFW3); 369 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) { 370 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN; 371 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) : 372 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN); 373 I915_WRITE(FW_BLC_SELF, val); 374 POSTING_READ(FW_BLC_SELF); 375 } else if (IS_I915GM(dev_priv)) { 376 /* 377 * FIXME can't find a bit like this for 915G, and 378 * and yet it does have the related watermark in 379 * FW_BLC_SELF. What's going on? 380 */ 381 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN; 382 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) : 383 _MASKED_BIT_DISABLE(INSTPM_SELF_EN); 384 I915_WRITE(INSTPM, val); 385 POSTING_READ(INSTPM); 386 } else { 387 return false; 388 } 389 390 trace_intel_memory_cxsr(dev_priv, was_enabled, enable); 391 392 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n", 393 enableddisabled(enable), 394 enableddisabled(was_enabled)); 395 396 return was_enabled; 397 } 398 399 /** 400 * intel_set_memory_cxsr - Configure CxSR state 401 * @dev_priv: i915 device 402 * @enable: Allow vs. disallow CxSR 403 * 404 * Allow or disallow the system to enter a special CxSR 405 * (C-state self refresh) state. What typically happens in CxSR mode 406 * is that several display FIFOs may get combined into a single larger 407 * FIFO for a particular plane (so called max FIFO mode) to allow the 408 * system to defer memory fetches longer, and the memory will enter 409 * self refresh. 410 * 411 * Note that enabling CxSR does not guarantee that the system enter 412 * this special mode, nor does it guarantee that the system stays 413 * in that mode once entered. So this just allows/disallows the system 414 * to autonomously utilize the CxSR mode. Other factors such as core 415 * C-states will affect when/if the system actually enters/exits the 416 * CxSR mode. 417 * 418 * Note that on VLV/CHV this actually only controls the max FIFO mode, 419 * and the system is free to enter/exit memory self refresh at any time 420 * even when the use of CxSR has been disallowed. 421 * 422 * While the system is actually in the CxSR/max FIFO mode, some plane 423 * control registers will not get latched on vblank. Thus in order to 424 * guarantee the system will respond to changes in the plane registers 425 * we must always disallow CxSR prior to making changes to those registers. 426 * Unfortunately the system will re-evaluate the CxSR conditions at 427 * frame start which happens after vblank start (which is when the plane 428 * registers would get latched), so we can't proceed with the plane update 429 * during the same frame where we disallowed CxSR. 430 * 431 * Certain platforms also have a deeper HPLL SR mode. Fortunately the 432 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold 433 * the hardware w.r.t. HPLL SR when writing to plane registers. 434 * Disallowing just CxSR is sufficient. 435 */ 436 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) 437 { 438 bool ret; 439 440 mutex_lock(&dev_priv->wm.wm_mutex); 441 ret = _intel_set_memory_cxsr(dev_priv, enable); 442 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 443 dev_priv->wm.vlv.cxsr = enable; 444 else if (IS_G4X(dev_priv)) 445 dev_priv->wm.g4x.cxsr = enable; 446 mutex_unlock(&dev_priv->wm.wm_mutex); 447 448 return ret; 449 } 450 451 /* 452 * Latency for FIFO fetches is dependent on several factors: 453 * - memory configuration (speed, channels) 454 * - chipset 455 * - current MCH state 456 * It can be fairly high in some situations, so here we assume a fairly 457 * pessimal value. It's a tradeoff between extra memory fetches (if we 458 * set this value too high, the FIFO will fetch frequently to stay full) 459 * and power consumption (set it too low to save power and we might see 460 * FIFO underruns and display "flicker"). 461 * 462 * A value of 5us seems to be a good balance; safe for very low end 463 * platforms but not overly aggressive on lower latency configs. 464 */ 465 static const int pessimal_latency_ns = 5000; 466 467 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \ 468 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8)) 469 470 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state) 471 { 472 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 473 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 474 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state; 475 enum pipe pipe = crtc->pipe; 476 int sprite0_start, sprite1_start; 477 478 switch (pipe) { 479 u32 dsparb, dsparb2, dsparb3; 480 case PIPE_A: 481 dsparb = I915_READ(DSPARB); 482 dsparb2 = I915_READ(DSPARB2); 483 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0); 484 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4); 485 break; 486 case PIPE_B: 487 dsparb = I915_READ(DSPARB); 488 dsparb2 = I915_READ(DSPARB2); 489 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8); 490 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12); 491 break; 492 case PIPE_C: 493 dsparb2 = I915_READ(DSPARB2); 494 dsparb3 = I915_READ(DSPARB3); 495 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16); 496 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20); 497 break; 498 default: 499 MISSING_CASE(pipe); 500 return; 501 } 502 503 fifo_state->plane[PLANE_PRIMARY] = sprite0_start; 504 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start; 505 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start; 506 fifo_state->plane[PLANE_CURSOR] = 63; 507 } 508 509 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv, 510 enum i9xx_plane_id i9xx_plane) 511 { 512 u32 dsparb = I915_READ(DSPARB); 513 int size; 514 515 size = dsparb & 0x7f; 516 if (i9xx_plane == PLANE_B) 517 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size; 518 519 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n", 520 dsparb, plane_name(i9xx_plane), size); 521 522 return size; 523 } 524 525 static int i830_get_fifo_size(struct drm_i915_private *dev_priv, 526 enum i9xx_plane_id i9xx_plane) 527 { 528 u32 dsparb = I915_READ(DSPARB); 529 int size; 530 531 size = dsparb & 0x1ff; 532 if (i9xx_plane == PLANE_B) 533 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size; 534 size >>= 1; /* Convert to cachelines */ 535 536 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n", 537 dsparb, plane_name(i9xx_plane), size); 538 539 return size; 540 } 541 542 static int i845_get_fifo_size(struct drm_i915_private *dev_priv, 543 enum i9xx_plane_id i9xx_plane) 544 { 545 u32 dsparb = I915_READ(DSPARB); 546 int size; 547 548 size = dsparb & 0x7f; 549 size >>= 2; /* Convert to cachelines */ 550 551 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n", 552 dsparb, plane_name(i9xx_plane), size); 553 554 return size; 555 } 556 557 /* Pineview has different values for various configs */ 558 static const struct intel_watermark_params pnv_display_wm = { 559 .fifo_size = PINEVIEW_DISPLAY_FIFO, 560 .max_wm = PINEVIEW_MAX_WM, 561 .default_wm = PINEVIEW_DFT_WM, 562 .guard_size = PINEVIEW_GUARD_WM, 563 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 564 }; 565 566 static const struct intel_watermark_params pnv_display_hplloff_wm = { 567 .fifo_size = PINEVIEW_DISPLAY_FIFO, 568 .max_wm = PINEVIEW_MAX_WM, 569 .default_wm = PINEVIEW_DFT_HPLLOFF_WM, 570 .guard_size = PINEVIEW_GUARD_WM, 571 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 572 }; 573 574 static const struct intel_watermark_params pnv_cursor_wm = { 575 .fifo_size = PINEVIEW_CURSOR_FIFO, 576 .max_wm = PINEVIEW_CURSOR_MAX_WM, 577 .default_wm = PINEVIEW_CURSOR_DFT_WM, 578 .guard_size = PINEVIEW_CURSOR_GUARD_WM, 579 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 580 }; 581 582 static const struct intel_watermark_params pnv_cursor_hplloff_wm = { 583 .fifo_size = PINEVIEW_CURSOR_FIFO, 584 .max_wm = PINEVIEW_CURSOR_MAX_WM, 585 .default_wm = PINEVIEW_CURSOR_DFT_WM, 586 .guard_size = PINEVIEW_CURSOR_GUARD_WM, 587 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 588 }; 589 590 static const struct intel_watermark_params i965_cursor_wm_info = { 591 .fifo_size = I965_CURSOR_FIFO, 592 .max_wm = I965_CURSOR_MAX_WM, 593 .default_wm = I965_CURSOR_DFT_WM, 594 .guard_size = 2, 595 .cacheline_size = I915_FIFO_LINE_SIZE, 596 }; 597 598 static const struct intel_watermark_params i945_wm_info = { 599 .fifo_size = I945_FIFO_SIZE, 600 .max_wm = I915_MAX_WM, 601 .default_wm = 1, 602 .guard_size = 2, 603 .cacheline_size = I915_FIFO_LINE_SIZE, 604 }; 605 606 static const struct intel_watermark_params i915_wm_info = { 607 .fifo_size = I915_FIFO_SIZE, 608 .max_wm = I915_MAX_WM, 609 .default_wm = 1, 610 .guard_size = 2, 611 .cacheline_size = I915_FIFO_LINE_SIZE, 612 }; 613 614 static const struct intel_watermark_params i830_a_wm_info = { 615 .fifo_size = I855GM_FIFO_SIZE, 616 .max_wm = I915_MAX_WM, 617 .default_wm = 1, 618 .guard_size = 2, 619 .cacheline_size = I830_FIFO_LINE_SIZE, 620 }; 621 622 static const struct intel_watermark_params i830_bc_wm_info = { 623 .fifo_size = I855GM_FIFO_SIZE, 624 .max_wm = I915_MAX_WM/2, 625 .default_wm = 1, 626 .guard_size = 2, 627 .cacheline_size = I830_FIFO_LINE_SIZE, 628 }; 629 630 static const struct intel_watermark_params i845_wm_info = { 631 .fifo_size = I830_FIFO_SIZE, 632 .max_wm = I915_MAX_WM, 633 .default_wm = 1, 634 .guard_size = 2, 635 .cacheline_size = I830_FIFO_LINE_SIZE, 636 }; 637 638 /** 639 * intel_wm_method1 - Method 1 / "small buffer" watermark formula 640 * @pixel_rate: Pipe pixel rate in kHz 641 * @cpp: Plane bytes per pixel 642 * @latency: Memory wakeup latency in 0.1us units 643 * 644 * Compute the watermark using the method 1 or "small buffer" 645 * formula. The caller may additonally add extra cachelines 646 * to account for TLB misses and clock crossings. 647 * 648 * This method is concerned with the short term drain rate 649 * of the FIFO, ie. it does not account for blanking periods 650 * which would effectively reduce the average drain rate across 651 * a longer period. The name "small" refers to the fact the 652 * FIFO is relatively small compared to the amount of data 653 * fetched. 654 * 655 * The FIFO level vs. time graph might look something like: 656 * 657 * |\ |\ 658 * | \ | \ 659 * __---__---__ (- plane active, _ blanking) 660 * -> time 661 * 662 * or perhaps like this: 663 * 664 * |\|\ |\|\ 665 * __----__----__ (- plane active, _ blanking) 666 * -> time 667 * 668 * Returns: 669 * The watermark in bytes 670 */ 671 static unsigned int intel_wm_method1(unsigned int pixel_rate, 672 unsigned int cpp, 673 unsigned int latency) 674 { 675 u64 ret; 676 677 ret = mul_u32_u32(pixel_rate, cpp * latency); 678 ret = DIV_ROUND_UP_ULL(ret, 10000); 679 680 return ret; 681 } 682 683 /** 684 * intel_wm_method2 - Method 2 / "large buffer" watermark formula 685 * @pixel_rate: Pipe pixel rate in kHz 686 * @htotal: Pipe horizontal total 687 * @width: Plane width in pixels 688 * @cpp: Plane bytes per pixel 689 * @latency: Memory wakeup latency in 0.1us units 690 * 691 * Compute the watermark using the method 2 or "large buffer" 692 * formula. The caller may additonally add extra cachelines 693 * to account for TLB misses and clock crossings. 694 * 695 * This method is concerned with the long term drain rate 696 * of the FIFO, ie. it does account for blanking periods 697 * which effectively reduce the average drain rate across 698 * a longer period. The name "large" refers to the fact the 699 * FIFO is relatively large compared to the amount of data 700 * fetched. 701 * 702 * The FIFO level vs. time graph might look something like: 703 * 704 * |\___ |\___ 705 * | \___ | \___ 706 * | \ | \ 707 * __ --__--__--__--__--__--__ (- plane active, _ blanking) 708 * -> time 709 * 710 * Returns: 711 * The watermark in bytes 712 */ 713 static unsigned int intel_wm_method2(unsigned int pixel_rate, 714 unsigned int htotal, 715 unsigned int width, 716 unsigned int cpp, 717 unsigned int latency) 718 { 719 unsigned int ret; 720 721 /* 722 * FIXME remove once all users are computing 723 * watermarks in the correct place. 724 */ 725 if (WARN_ON_ONCE(htotal == 0)) 726 htotal = 1; 727 728 ret = (latency * pixel_rate) / (htotal * 10000); 729 ret = (ret + 1) * width * cpp; 730 731 return ret; 732 } 733 734 /** 735 * intel_calculate_wm - calculate watermark level 736 * @pixel_rate: pixel clock 737 * @wm: chip FIFO params 738 * @fifo_size: size of the FIFO buffer 739 * @cpp: bytes per pixel 740 * @latency_ns: memory latency for the platform 741 * 742 * Calculate the watermark level (the level at which the display plane will 743 * start fetching from memory again). Each chip has a different display 744 * FIFO size and allocation, so the caller needs to figure that out and pass 745 * in the correct intel_watermark_params structure. 746 * 747 * As the pixel clock runs, the FIFO will be drained at a rate that depends 748 * on the pixel size. When it reaches the watermark level, it'll start 749 * fetching FIFO line sized based chunks from memory until the FIFO fills 750 * past the watermark point. If the FIFO drains completely, a FIFO underrun 751 * will occur, and a display engine hang could result. 752 */ 753 static unsigned int intel_calculate_wm(int pixel_rate, 754 const struct intel_watermark_params *wm, 755 int fifo_size, int cpp, 756 unsigned int latency_ns) 757 { 758 int entries, wm_size; 759 760 /* 761 * Note: we need to make sure we don't overflow for various clock & 762 * latency values. 763 * clocks go from a few thousand to several hundred thousand. 764 * latency is usually a few thousand 765 */ 766 entries = intel_wm_method1(pixel_rate, cpp, 767 latency_ns / 100); 768 entries = DIV_ROUND_UP(entries, wm->cacheline_size) + 769 wm->guard_size; 770 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries); 771 772 wm_size = fifo_size - entries; 773 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size); 774 775 /* Don't promote wm_size to unsigned... */ 776 if (wm_size > wm->max_wm) 777 wm_size = wm->max_wm; 778 if (wm_size <= 0) 779 wm_size = wm->default_wm; 780 781 /* 782 * Bspec seems to indicate that the value shouldn't be lower than 783 * 'burst size + 1'. Certainly 830 is quite unhappy with low values. 784 * Lets go for 8 which is the burst size since certain platforms 785 * already use a hardcoded 8 (which is what the spec says should be 786 * done). 787 */ 788 if (wm_size <= 8) 789 wm_size = 8; 790 791 return wm_size; 792 } 793 794 static bool is_disabling(int old, int new, int threshold) 795 { 796 return old >= threshold && new < threshold; 797 } 798 799 static bool is_enabling(int old, int new, int threshold) 800 { 801 return old < threshold && new >= threshold; 802 } 803 804 static int intel_wm_num_levels(struct drm_i915_private *dev_priv) 805 { 806 return dev_priv->wm.max_level + 1; 807 } 808 809 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state, 810 const struct intel_plane_state *plane_state) 811 { 812 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 813 814 /* FIXME check the 'enable' instead */ 815 if (!crtc_state->hw.active) 816 return false; 817 818 /* 819 * Treat cursor with fb as always visible since cursor updates 820 * can happen faster than the vrefresh rate, and the current 821 * watermark code doesn't handle that correctly. Cursor updates 822 * which set/clear the fb or change the cursor size are going 823 * to get throttled by intel_legacy_cursor_update() to work 824 * around this problem with the watermark code. 825 */ 826 if (plane->id == PLANE_CURSOR) 827 return plane_state->hw.fb != NULL; 828 else 829 return plane_state->uapi.visible; 830 } 831 832 static bool intel_crtc_active(struct intel_crtc *crtc) 833 { 834 /* Be paranoid as we can arrive here with only partial 835 * state retrieved from the hardware during setup. 836 * 837 * We can ditch the adjusted_mode.crtc_clock check as soon 838 * as Haswell has gained clock readout/fastboot support. 839 * 840 * We can ditch the crtc->primary->state->fb check as soon as we can 841 * properly reconstruct framebuffers. 842 * 843 * FIXME: The intel_crtc->active here should be switched to 844 * crtc->state->active once we have proper CRTC states wired up 845 * for atomic. 846 */ 847 return crtc->active && crtc->base.primary->state->fb && 848 crtc->config->hw.adjusted_mode.crtc_clock; 849 } 850 851 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv) 852 { 853 struct intel_crtc *crtc, *enabled = NULL; 854 855 for_each_intel_crtc(&dev_priv->drm, crtc) { 856 if (intel_crtc_active(crtc)) { 857 if (enabled) 858 return NULL; 859 enabled = crtc; 860 } 861 } 862 863 return enabled; 864 } 865 866 static void pnv_update_wm(struct intel_crtc *unused_crtc) 867 { 868 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); 869 struct intel_crtc *crtc; 870 const struct cxsr_latency *latency; 871 u32 reg; 872 unsigned int wm; 873 874 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv), 875 dev_priv->is_ddr3, 876 dev_priv->fsb_freq, 877 dev_priv->mem_freq); 878 if (!latency) { 879 drm_dbg_kms(&dev_priv->drm, 880 "Unknown FSB/MEM found, disable CxSR\n"); 881 intel_set_memory_cxsr(dev_priv, false); 882 return; 883 } 884 885 crtc = single_enabled_crtc(dev_priv); 886 if (crtc) { 887 const struct drm_display_mode *adjusted_mode = 888 &crtc->config->hw.adjusted_mode; 889 const struct drm_framebuffer *fb = 890 crtc->base.primary->state->fb; 891 int cpp = fb->format->cpp[0]; 892 int clock = adjusted_mode->crtc_clock; 893 894 /* Display SR */ 895 wm = intel_calculate_wm(clock, &pnv_display_wm, 896 pnv_display_wm.fifo_size, 897 cpp, latency->display_sr); 898 reg = I915_READ(DSPFW1); 899 reg &= ~DSPFW_SR_MASK; 900 reg |= FW_WM(wm, SR); 901 I915_WRITE(DSPFW1, reg); 902 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg); 903 904 /* cursor SR */ 905 wm = intel_calculate_wm(clock, &pnv_cursor_wm, 906 pnv_display_wm.fifo_size, 907 4, latency->cursor_sr); 908 reg = I915_READ(DSPFW3); 909 reg &= ~DSPFW_CURSOR_SR_MASK; 910 reg |= FW_WM(wm, CURSOR_SR); 911 I915_WRITE(DSPFW3, reg); 912 913 /* Display HPLL off SR */ 914 wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm, 915 pnv_display_hplloff_wm.fifo_size, 916 cpp, latency->display_hpll_disable); 917 reg = I915_READ(DSPFW3); 918 reg &= ~DSPFW_HPLL_SR_MASK; 919 reg |= FW_WM(wm, HPLL_SR); 920 I915_WRITE(DSPFW3, reg); 921 922 /* cursor HPLL off SR */ 923 wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm, 924 pnv_display_hplloff_wm.fifo_size, 925 4, latency->cursor_hpll_disable); 926 reg = I915_READ(DSPFW3); 927 reg &= ~DSPFW_HPLL_CURSOR_MASK; 928 reg |= FW_WM(wm, HPLL_CURSOR); 929 I915_WRITE(DSPFW3, reg); 930 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg); 931 932 intel_set_memory_cxsr(dev_priv, true); 933 } else { 934 intel_set_memory_cxsr(dev_priv, false); 935 } 936 } 937 938 /* 939 * Documentation says: 940 * "If the line size is small, the TLB fetches can get in the way of the 941 * data fetches, causing some lag in the pixel data return which is not 942 * accounted for in the above formulas. The following adjustment only 943 * needs to be applied if eight whole lines fit in the buffer at once. 944 * The WM is adjusted upwards by the difference between the FIFO size 945 * and the size of 8 whole lines. This adjustment is always performed 946 * in the actual pixel depth regardless of whether FBC is enabled or not." 947 */ 948 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp) 949 { 950 int tlb_miss = fifo_size * 64 - width * cpp * 8; 951 952 return max(0, tlb_miss); 953 } 954 955 static void g4x_write_wm_values(struct drm_i915_private *dev_priv, 956 const struct g4x_wm_values *wm) 957 { 958 enum pipe pipe; 959 960 for_each_pipe(dev_priv, pipe) 961 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm); 962 963 I915_WRITE(DSPFW1, 964 FW_WM(wm->sr.plane, SR) | 965 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) | 966 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) | 967 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA)); 968 I915_WRITE(DSPFW2, 969 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) | 970 FW_WM(wm->sr.fbc, FBC_SR) | 971 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) | 972 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) | 973 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) | 974 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA)); 975 I915_WRITE(DSPFW3, 976 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) | 977 FW_WM(wm->sr.cursor, CURSOR_SR) | 978 FW_WM(wm->hpll.cursor, HPLL_CURSOR) | 979 FW_WM(wm->hpll.plane, HPLL_SR)); 980 981 POSTING_READ(DSPFW1); 982 } 983 984 #define FW_WM_VLV(value, plane) \ 985 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV) 986 987 static void vlv_write_wm_values(struct drm_i915_private *dev_priv, 988 const struct vlv_wm_values *wm) 989 { 990 enum pipe pipe; 991 992 for_each_pipe(dev_priv, pipe) { 993 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm); 994 995 I915_WRITE(VLV_DDL(pipe), 996 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) | 997 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) | 998 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) | 999 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT)); 1000 } 1001 1002 /* 1003 * Zero the (unused) WM1 watermarks, and also clear all the 1004 * high order bits so that there are no out of bounds values 1005 * present in the registers during the reprogramming. 1006 */ 1007 I915_WRITE(DSPHOWM, 0); 1008 I915_WRITE(DSPHOWM1, 0); 1009 I915_WRITE(DSPFW4, 0); 1010 I915_WRITE(DSPFW5, 0); 1011 I915_WRITE(DSPFW6, 0); 1012 1013 I915_WRITE(DSPFW1, 1014 FW_WM(wm->sr.plane, SR) | 1015 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) | 1016 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) | 1017 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA)); 1018 I915_WRITE(DSPFW2, 1019 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) | 1020 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) | 1021 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA)); 1022 I915_WRITE(DSPFW3, 1023 FW_WM(wm->sr.cursor, CURSOR_SR)); 1024 1025 if (IS_CHERRYVIEW(dev_priv)) { 1026 I915_WRITE(DSPFW7_CHV, 1027 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) | 1028 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC)); 1029 I915_WRITE(DSPFW8_CHV, 1030 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) | 1031 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE)); 1032 I915_WRITE(DSPFW9_CHV, 1033 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) | 1034 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC)); 1035 I915_WRITE(DSPHOWM, 1036 FW_WM(wm->sr.plane >> 9, SR_HI) | 1037 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) | 1038 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) | 1039 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) | 1040 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) | 1041 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) | 1042 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) | 1043 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) | 1044 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) | 1045 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI)); 1046 } else { 1047 I915_WRITE(DSPFW7, 1048 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) | 1049 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC)); 1050 I915_WRITE(DSPHOWM, 1051 FW_WM(wm->sr.plane >> 9, SR_HI) | 1052 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) | 1053 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) | 1054 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) | 1055 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) | 1056 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) | 1057 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI)); 1058 } 1059 1060 POSTING_READ(DSPFW1); 1061 } 1062 1063 #undef FW_WM_VLV 1064 1065 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv) 1066 { 1067 /* all latencies in usec */ 1068 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5; 1069 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12; 1070 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35; 1071 1072 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL; 1073 } 1074 1075 static int g4x_plane_fifo_size(enum plane_id plane_id, int level) 1076 { 1077 /* 1078 * DSPCNTR[13] supposedly controls whether the 1079 * primary plane can use the FIFO space otherwise 1080 * reserved for the sprite plane. It's not 100% clear 1081 * what the actual FIFO size is, but it looks like we 1082 * can happily set both primary and sprite watermarks 1083 * up to 127 cachelines. So that would seem to mean 1084 * that either DSPCNTR[13] doesn't do anything, or that 1085 * the total FIFO is >= 256 cachelines in size. Either 1086 * way, we don't seem to have to worry about this 1087 * repartitioning as the maximum watermark value the 1088 * register can hold for each plane is lower than the 1089 * minimum FIFO size. 1090 */ 1091 switch (plane_id) { 1092 case PLANE_CURSOR: 1093 return 63; 1094 case PLANE_PRIMARY: 1095 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511; 1096 case PLANE_SPRITE0: 1097 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0; 1098 default: 1099 MISSING_CASE(plane_id); 1100 return 0; 1101 } 1102 } 1103 1104 static int g4x_fbc_fifo_size(int level) 1105 { 1106 switch (level) { 1107 case G4X_WM_LEVEL_SR: 1108 return 7; 1109 case G4X_WM_LEVEL_HPLL: 1110 return 15; 1111 default: 1112 MISSING_CASE(level); 1113 return 0; 1114 } 1115 } 1116 1117 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state, 1118 const struct intel_plane_state *plane_state, 1119 int level) 1120 { 1121 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 1122 struct drm_i915_private *dev_priv = to_i915(plane->base.dev); 1123 const struct drm_display_mode *adjusted_mode = 1124 &crtc_state->hw.adjusted_mode; 1125 unsigned int latency = dev_priv->wm.pri_latency[level] * 10; 1126 unsigned int clock, htotal, cpp, width, wm; 1127 1128 if (latency == 0) 1129 return USHRT_MAX; 1130 1131 if (!intel_wm_plane_visible(crtc_state, plane_state)) 1132 return 0; 1133 1134 cpp = plane_state->hw.fb->format->cpp[0]; 1135 1136 /* 1137 * Not 100% sure which way ELK should go here as the 1138 * spec only says CL/CTG should assume 32bpp and BW 1139 * doesn't need to. But as these things followed the 1140 * mobile vs. desktop lines on gen3 as well, let's 1141 * assume ELK doesn't need this. 1142 * 1143 * The spec also fails to list such a restriction for 1144 * the HPLL watermark, which seems a little strange. 1145 * Let's use 32bpp for the HPLL watermark as well. 1146 */ 1147 if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY && 1148 level != G4X_WM_LEVEL_NORMAL) 1149 cpp = max(cpp, 4u); 1150 1151 clock = adjusted_mode->crtc_clock; 1152 htotal = adjusted_mode->crtc_htotal; 1153 1154 width = drm_rect_width(&plane_state->uapi.dst); 1155 1156 if (plane->id == PLANE_CURSOR) { 1157 wm = intel_wm_method2(clock, htotal, width, cpp, latency); 1158 } else if (plane->id == PLANE_PRIMARY && 1159 level == G4X_WM_LEVEL_NORMAL) { 1160 wm = intel_wm_method1(clock, cpp, latency); 1161 } else { 1162 unsigned int small, large; 1163 1164 small = intel_wm_method1(clock, cpp, latency); 1165 large = intel_wm_method2(clock, htotal, width, cpp, latency); 1166 1167 wm = min(small, large); 1168 } 1169 1170 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level), 1171 width, cpp); 1172 1173 wm = DIV_ROUND_UP(wm, 64) + 2; 1174 1175 return min_t(unsigned int, wm, USHRT_MAX); 1176 } 1177 1178 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state, 1179 int level, enum plane_id plane_id, u16 value) 1180 { 1181 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 1182 bool dirty = false; 1183 1184 for (; level < intel_wm_num_levels(dev_priv); level++) { 1185 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level]; 1186 1187 dirty |= raw->plane[plane_id] != value; 1188 raw->plane[plane_id] = value; 1189 } 1190 1191 return dirty; 1192 } 1193 1194 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state, 1195 int level, u16 value) 1196 { 1197 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 1198 bool dirty = false; 1199 1200 /* NORMAL level doesn't have an FBC watermark */ 1201 level = max(level, G4X_WM_LEVEL_SR); 1202 1203 for (; level < intel_wm_num_levels(dev_priv); level++) { 1204 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level]; 1205 1206 dirty |= raw->fbc != value; 1207 raw->fbc = value; 1208 } 1209 1210 return dirty; 1211 } 1212 1213 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state, 1214 const struct intel_plane_state *plane_state, 1215 u32 pri_val); 1216 1217 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state, 1218 const struct intel_plane_state *plane_state) 1219 { 1220 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 1221 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 1222 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev)); 1223 enum plane_id plane_id = plane->id; 1224 bool dirty = false; 1225 int level; 1226 1227 if (!intel_wm_plane_visible(crtc_state, plane_state)) { 1228 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0); 1229 if (plane_id == PLANE_PRIMARY) 1230 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0); 1231 goto out; 1232 } 1233 1234 for (level = 0; level < num_levels; level++) { 1235 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level]; 1236 int wm, max_wm; 1237 1238 wm = g4x_compute_wm(crtc_state, plane_state, level); 1239 max_wm = g4x_plane_fifo_size(plane_id, level); 1240 1241 if (wm > max_wm) 1242 break; 1243 1244 dirty |= raw->plane[plane_id] != wm; 1245 raw->plane[plane_id] = wm; 1246 1247 if (plane_id != PLANE_PRIMARY || 1248 level == G4X_WM_LEVEL_NORMAL) 1249 continue; 1250 1251 wm = ilk_compute_fbc_wm(crtc_state, plane_state, 1252 raw->plane[plane_id]); 1253 max_wm = g4x_fbc_fifo_size(level); 1254 1255 /* 1256 * FBC wm is not mandatory as we 1257 * can always just disable its use. 1258 */ 1259 if (wm > max_wm) 1260 wm = USHRT_MAX; 1261 1262 dirty |= raw->fbc != wm; 1263 raw->fbc = wm; 1264 } 1265 1266 /* mark watermarks as invalid */ 1267 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX); 1268 1269 if (plane_id == PLANE_PRIMARY) 1270 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX); 1271 1272 out: 1273 if (dirty) { 1274 drm_dbg_kms(&dev_priv->drm, 1275 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n", 1276 plane->base.name, 1277 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id], 1278 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id], 1279 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]); 1280 1281 if (plane_id == PLANE_PRIMARY) 1282 drm_dbg_kms(&dev_priv->drm, 1283 "FBC watermarks: SR=%d, HPLL=%d\n", 1284 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc, 1285 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc); 1286 } 1287 1288 return dirty; 1289 } 1290 1291 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state, 1292 enum plane_id plane_id, int level) 1293 { 1294 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level]; 1295 1296 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level); 1297 } 1298 1299 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, 1300 int level) 1301 { 1302 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 1303 1304 if (level > dev_priv->wm.max_level) 1305 return false; 1306 1307 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) && 1308 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) && 1309 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level); 1310 } 1311 1312 /* mark all levels starting from 'level' as invalid */ 1313 static void g4x_invalidate_wms(struct intel_crtc *crtc, 1314 struct g4x_wm_state *wm_state, int level) 1315 { 1316 if (level <= G4X_WM_LEVEL_NORMAL) { 1317 enum plane_id plane_id; 1318 1319 for_each_plane_id_on_crtc(crtc, plane_id) 1320 wm_state->wm.plane[plane_id] = USHRT_MAX; 1321 } 1322 1323 if (level <= G4X_WM_LEVEL_SR) { 1324 wm_state->cxsr = false; 1325 wm_state->sr.cursor = USHRT_MAX; 1326 wm_state->sr.plane = USHRT_MAX; 1327 wm_state->sr.fbc = USHRT_MAX; 1328 } 1329 1330 if (level <= G4X_WM_LEVEL_HPLL) { 1331 wm_state->hpll_en = false; 1332 wm_state->hpll.cursor = USHRT_MAX; 1333 wm_state->hpll.plane = USHRT_MAX; 1334 wm_state->hpll.fbc = USHRT_MAX; 1335 } 1336 } 1337 1338 static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state) 1339 { 1340 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 1341 struct intel_atomic_state *state = 1342 to_intel_atomic_state(crtc_state->uapi.state); 1343 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal; 1344 int num_active_planes = hweight8(crtc_state->active_planes & 1345 ~BIT(PLANE_CURSOR)); 1346 const struct g4x_pipe_wm *raw; 1347 const struct intel_plane_state *old_plane_state; 1348 const struct intel_plane_state *new_plane_state; 1349 struct intel_plane *plane; 1350 enum plane_id plane_id; 1351 int i, level; 1352 unsigned int dirty = 0; 1353 1354 for_each_oldnew_intel_plane_in_state(state, plane, 1355 old_plane_state, 1356 new_plane_state, i) { 1357 if (new_plane_state->hw.crtc != &crtc->base && 1358 old_plane_state->hw.crtc != &crtc->base) 1359 continue; 1360 1361 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state)) 1362 dirty |= BIT(plane->id); 1363 } 1364 1365 if (!dirty) 1366 return 0; 1367 1368 level = G4X_WM_LEVEL_NORMAL; 1369 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level)) 1370 goto out; 1371 1372 raw = &crtc_state->wm.g4x.raw[level]; 1373 for_each_plane_id_on_crtc(crtc, plane_id) 1374 wm_state->wm.plane[plane_id] = raw->plane[plane_id]; 1375 1376 level = G4X_WM_LEVEL_SR; 1377 1378 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level)) 1379 goto out; 1380 1381 raw = &crtc_state->wm.g4x.raw[level]; 1382 wm_state->sr.plane = raw->plane[PLANE_PRIMARY]; 1383 wm_state->sr.cursor = raw->plane[PLANE_CURSOR]; 1384 wm_state->sr.fbc = raw->fbc; 1385 1386 wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY); 1387 1388 level = G4X_WM_LEVEL_HPLL; 1389 1390 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level)) 1391 goto out; 1392 1393 raw = &crtc_state->wm.g4x.raw[level]; 1394 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY]; 1395 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR]; 1396 wm_state->hpll.fbc = raw->fbc; 1397 1398 wm_state->hpll_en = wm_state->cxsr; 1399 1400 level++; 1401 1402 out: 1403 if (level == G4X_WM_LEVEL_NORMAL) 1404 return -EINVAL; 1405 1406 /* invalidate the higher levels */ 1407 g4x_invalidate_wms(crtc, wm_state, level); 1408 1409 /* 1410 * Determine if the FBC watermark(s) can be used. IF 1411 * this isn't the case we prefer to disable the FBC 1412 ( watermark(s) rather than disable the SR/HPLL 1413 * level(s) entirely. 1414 */ 1415 wm_state->fbc_en = level > G4X_WM_LEVEL_NORMAL; 1416 1417 if (level >= G4X_WM_LEVEL_SR && 1418 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR)) 1419 wm_state->fbc_en = false; 1420 else if (level >= G4X_WM_LEVEL_HPLL && 1421 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL)) 1422 wm_state->fbc_en = false; 1423 1424 return 0; 1425 } 1426 1427 static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state) 1428 { 1429 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); 1430 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate; 1431 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal; 1432 struct intel_atomic_state *intel_state = 1433 to_intel_atomic_state(new_crtc_state->uapi.state); 1434 const struct intel_crtc_state *old_crtc_state = 1435 intel_atomic_get_old_crtc_state(intel_state, crtc); 1436 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal; 1437 enum plane_id plane_id; 1438 1439 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) { 1440 *intermediate = *optimal; 1441 1442 intermediate->cxsr = false; 1443 intermediate->hpll_en = false; 1444 goto out; 1445 } 1446 1447 intermediate->cxsr = optimal->cxsr && active->cxsr && 1448 !new_crtc_state->disable_cxsr; 1449 intermediate->hpll_en = optimal->hpll_en && active->hpll_en && 1450 !new_crtc_state->disable_cxsr; 1451 intermediate->fbc_en = optimal->fbc_en && active->fbc_en; 1452 1453 for_each_plane_id_on_crtc(crtc, plane_id) { 1454 intermediate->wm.plane[plane_id] = 1455 max(optimal->wm.plane[plane_id], 1456 active->wm.plane[plane_id]); 1457 1458 WARN_ON(intermediate->wm.plane[plane_id] > 1459 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL)); 1460 } 1461 1462 intermediate->sr.plane = max(optimal->sr.plane, 1463 active->sr.plane); 1464 intermediate->sr.cursor = max(optimal->sr.cursor, 1465 active->sr.cursor); 1466 intermediate->sr.fbc = max(optimal->sr.fbc, 1467 active->sr.fbc); 1468 1469 intermediate->hpll.plane = max(optimal->hpll.plane, 1470 active->hpll.plane); 1471 intermediate->hpll.cursor = max(optimal->hpll.cursor, 1472 active->hpll.cursor); 1473 intermediate->hpll.fbc = max(optimal->hpll.fbc, 1474 active->hpll.fbc); 1475 1476 WARN_ON((intermediate->sr.plane > 1477 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) || 1478 intermediate->sr.cursor > 1479 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) && 1480 intermediate->cxsr); 1481 WARN_ON((intermediate->sr.plane > 1482 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) || 1483 intermediate->sr.cursor > 1484 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) && 1485 intermediate->hpll_en); 1486 1487 WARN_ON(intermediate->sr.fbc > g4x_fbc_fifo_size(1) && 1488 intermediate->fbc_en && intermediate->cxsr); 1489 WARN_ON(intermediate->hpll.fbc > g4x_fbc_fifo_size(2) && 1490 intermediate->fbc_en && intermediate->hpll_en); 1491 1492 out: 1493 /* 1494 * If our intermediate WM are identical to the final WM, then we can 1495 * omit the post-vblank programming; only update if it's different. 1496 */ 1497 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0) 1498 new_crtc_state->wm.need_postvbl_update = true; 1499 1500 return 0; 1501 } 1502 1503 static void g4x_merge_wm(struct drm_i915_private *dev_priv, 1504 struct g4x_wm_values *wm) 1505 { 1506 struct intel_crtc *crtc; 1507 int num_active_pipes = 0; 1508 1509 wm->cxsr = true; 1510 wm->hpll_en = true; 1511 wm->fbc_en = true; 1512 1513 for_each_intel_crtc(&dev_priv->drm, crtc) { 1514 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x; 1515 1516 if (!crtc->active) 1517 continue; 1518 1519 if (!wm_state->cxsr) 1520 wm->cxsr = false; 1521 if (!wm_state->hpll_en) 1522 wm->hpll_en = false; 1523 if (!wm_state->fbc_en) 1524 wm->fbc_en = false; 1525 1526 num_active_pipes++; 1527 } 1528 1529 if (num_active_pipes != 1) { 1530 wm->cxsr = false; 1531 wm->hpll_en = false; 1532 wm->fbc_en = false; 1533 } 1534 1535 for_each_intel_crtc(&dev_priv->drm, crtc) { 1536 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x; 1537 enum pipe pipe = crtc->pipe; 1538 1539 wm->pipe[pipe] = wm_state->wm; 1540 if (crtc->active && wm->cxsr) 1541 wm->sr = wm_state->sr; 1542 if (crtc->active && wm->hpll_en) 1543 wm->hpll = wm_state->hpll; 1544 } 1545 } 1546 1547 static void g4x_program_watermarks(struct drm_i915_private *dev_priv) 1548 { 1549 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x; 1550 struct g4x_wm_values new_wm = {}; 1551 1552 g4x_merge_wm(dev_priv, &new_wm); 1553 1554 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0) 1555 return; 1556 1557 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true)) 1558 _intel_set_memory_cxsr(dev_priv, false); 1559 1560 g4x_write_wm_values(dev_priv, &new_wm); 1561 1562 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true)) 1563 _intel_set_memory_cxsr(dev_priv, true); 1564 1565 *old_wm = new_wm; 1566 } 1567 1568 static void g4x_initial_watermarks(struct intel_atomic_state *state, 1569 struct intel_crtc *crtc) 1570 { 1571 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 1572 const struct intel_crtc_state *crtc_state = 1573 intel_atomic_get_new_crtc_state(state, crtc); 1574 1575 mutex_lock(&dev_priv->wm.wm_mutex); 1576 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate; 1577 g4x_program_watermarks(dev_priv); 1578 mutex_unlock(&dev_priv->wm.wm_mutex); 1579 } 1580 1581 static void g4x_optimize_watermarks(struct intel_atomic_state *state, 1582 struct intel_crtc *crtc) 1583 { 1584 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 1585 const struct intel_crtc_state *crtc_state = 1586 intel_atomic_get_new_crtc_state(state, crtc); 1587 1588 if (!crtc_state->wm.need_postvbl_update) 1589 return; 1590 1591 mutex_lock(&dev_priv->wm.wm_mutex); 1592 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal; 1593 g4x_program_watermarks(dev_priv); 1594 mutex_unlock(&dev_priv->wm.wm_mutex); 1595 } 1596 1597 /* latency must be in 0.1us units. */ 1598 static unsigned int vlv_wm_method2(unsigned int pixel_rate, 1599 unsigned int htotal, 1600 unsigned int width, 1601 unsigned int cpp, 1602 unsigned int latency) 1603 { 1604 unsigned int ret; 1605 1606 ret = intel_wm_method2(pixel_rate, htotal, 1607 width, cpp, latency); 1608 ret = DIV_ROUND_UP(ret, 64); 1609 1610 return ret; 1611 } 1612 1613 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv) 1614 { 1615 /* all latencies in usec */ 1616 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3; 1617 1618 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2; 1619 1620 if (IS_CHERRYVIEW(dev_priv)) { 1621 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12; 1622 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33; 1623 1624 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS; 1625 } 1626 } 1627 1628 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state, 1629 const struct intel_plane_state *plane_state, 1630 int level) 1631 { 1632 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 1633 struct drm_i915_private *dev_priv = to_i915(plane->base.dev); 1634 const struct drm_display_mode *adjusted_mode = 1635 &crtc_state->hw.adjusted_mode; 1636 unsigned int clock, htotal, cpp, width, wm; 1637 1638 if (dev_priv->wm.pri_latency[level] == 0) 1639 return USHRT_MAX; 1640 1641 if (!intel_wm_plane_visible(crtc_state, plane_state)) 1642 return 0; 1643 1644 cpp = plane_state->hw.fb->format->cpp[0]; 1645 clock = adjusted_mode->crtc_clock; 1646 htotal = adjusted_mode->crtc_htotal; 1647 width = crtc_state->pipe_src_w; 1648 1649 if (plane->id == PLANE_CURSOR) { 1650 /* 1651 * FIXME the formula gives values that are 1652 * too big for the cursor FIFO, and hence we 1653 * would never be able to use cursors. For 1654 * now just hardcode the watermark. 1655 */ 1656 wm = 63; 1657 } else { 1658 wm = vlv_wm_method2(clock, htotal, width, cpp, 1659 dev_priv->wm.pri_latency[level] * 10); 1660 } 1661 1662 return min_t(unsigned int, wm, USHRT_MAX); 1663 } 1664 1665 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes) 1666 { 1667 return (active_planes & (BIT(PLANE_SPRITE0) | 1668 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1); 1669 } 1670 1671 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state) 1672 { 1673 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 1674 const struct g4x_pipe_wm *raw = 1675 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2]; 1676 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state; 1677 unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR); 1678 int num_active_planes = hweight8(active_planes); 1679 const int fifo_size = 511; 1680 int fifo_extra, fifo_left = fifo_size; 1681 int sprite0_fifo_extra = 0; 1682 unsigned int total_rate; 1683 enum plane_id plane_id; 1684 1685 /* 1686 * When enabling sprite0 after sprite1 has already been enabled 1687 * we tend to get an underrun unless sprite0 already has some 1688 * FIFO space allcoated. Hence we always allocate at least one 1689 * cacheline for sprite0 whenever sprite1 is enabled. 1690 * 1691 * All other plane enable sequences appear immune to this problem. 1692 */ 1693 if (vlv_need_sprite0_fifo_workaround(active_planes)) 1694 sprite0_fifo_extra = 1; 1695 1696 total_rate = raw->plane[PLANE_PRIMARY] + 1697 raw->plane[PLANE_SPRITE0] + 1698 raw->plane[PLANE_SPRITE1] + 1699 sprite0_fifo_extra; 1700 1701 if (total_rate > fifo_size) 1702 return -EINVAL; 1703 1704 if (total_rate == 0) 1705 total_rate = 1; 1706 1707 for_each_plane_id_on_crtc(crtc, plane_id) { 1708 unsigned int rate; 1709 1710 if ((active_planes & BIT(plane_id)) == 0) { 1711 fifo_state->plane[plane_id] = 0; 1712 continue; 1713 } 1714 1715 rate = raw->plane[plane_id]; 1716 fifo_state->plane[plane_id] = fifo_size * rate / total_rate; 1717 fifo_left -= fifo_state->plane[plane_id]; 1718 } 1719 1720 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra; 1721 fifo_left -= sprite0_fifo_extra; 1722 1723 fifo_state->plane[PLANE_CURSOR] = 63; 1724 1725 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1); 1726 1727 /* spread the remainder evenly */ 1728 for_each_plane_id_on_crtc(crtc, plane_id) { 1729 int plane_extra; 1730 1731 if (fifo_left == 0) 1732 break; 1733 1734 if ((active_planes & BIT(plane_id)) == 0) 1735 continue; 1736 1737 plane_extra = min(fifo_extra, fifo_left); 1738 fifo_state->plane[plane_id] += plane_extra; 1739 fifo_left -= plane_extra; 1740 } 1741 1742 WARN_ON(active_planes != 0 && fifo_left != 0); 1743 1744 /* give it all to the first plane if none are active */ 1745 if (active_planes == 0) { 1746 WARN_ON(fifo_left != fifo_size); 1747 fifo_state->plane[PLANE_PRIMARY] = fifo_left; 1748 } 1749 1750 return 0; 1751 } 1752 1753 /* mark all levels starting from 'level' as invalid */ 1754 static void vlv_invalidate_wms(struct intel_crtc *crtc, 1755 struct vlv_wm_state *wm_state, int level) 1756 { 1757 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 1758 1759 for (; level < intel_wm_num_levels(dev_priv); level++) { 1760 enum plane_id plane_id; 1761 1762 for_each_plane_id_on_crtc(crtc, plane_id) 1763 wm_state->wm[level].plane[plane_id] = USHRT_MAX; 1764 1765 wm_state->sr[level].cursor = USHRT_MAX; 1766 wm_state->sr[level].plane = USHRT_MAX; 1767 } 1768 } 1769 1770 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size) 1771 { 1772 if (wm > fifo_size) 1773 return USHRT_MAX; 1774 else 1775 return fifo_size - wm; 1776 } 1777 1778 /* 1779 * Starting from 'level' set all higher 1780 * levels to 'value' in the "raw" watermarks. 1781 */ 1782 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state, 1783 int level, enum plane_id plane_id, u16 value) 1784 { 1785 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 1786 int num_levels = intel_wm_num_levels(dev_priv); 1787 bool dirty = false; 1788 1789 for (; level < num_levels; level++) { 1790 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level]; 1791 1792 dirty |= raw->plane[plane_id] != value; 1793 raw->plane[plane_id] = value; 1794 } 1795 1796 return dirty; 1797 } 1798 1799 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state, 1800 const struct intel_plane_state *plane_state) 1801 { 1802 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 1803 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 1804 enum plane_id plane_id = plane->id; 1805 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev)); 1806 int level; 1807 bool dirty = false; 1808 1809 if (!intel_wm_plane_visible(crtc_state, plane_state)) { 1810 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0); 1811 goto out; 1812 } 1813 1814 for (level = 0; level < num_levels; level++) { 1815 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level]; 1816 int wm = vlv_compute_wm_level(crtc_state, plane_state, level); 1817 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511; 1818 1819 if (wm > max_wm) 1820 break; 1821 1822 dirty |= raw->plane[plane_id] != wm; 1823 raw->plane[plane_id] = wm; 1824 } 1825 1826 /* mark all higher levels as invalid */ 1827 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX); 1828 1829 out: 1830 if (dirty) 1831 drm_dbg_kms(&dev_priv->drm, 1832 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n", 1833 plane->base.name, 1834 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id], 1835 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id], 1836 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]); 1837 1838 return dirty; 1839 } 1840 1841 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state, 1842 enum plane_id plane_id, int level) 1843 { 1844 const struct g4x_pipe_wm *raw = 1845 &crtc_state->wm.vlv.raw[level]; 1846 const struct vlv_fifo_state *fifo_state = 1847 &crtc_state->wm.vlv.fifo_state; 1848 1849 return raw->plane[plane_id] <= fifo_state->plane[plane_id]; 1850 } 1851 1852 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level) 1853 { 1854 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) && 1855 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) && 1856 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) && 1857 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level); 1858 } 1859 1860 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state) 1861 { 1862 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 1863 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 1864 struct intel_atomic_state *state = 1865 to_intel_atomic_state(crtc_state->uapi.state); 1866 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal; 1867 const struct vlv_fifo_state *fifo_state = 1868 &crtc_state->wm.vlv.fifo_state; 1869 int num_active_planes = hweight8(crtc_state->active_planes & 1870 ~BIT(PLANE_CURSOR)); 1871 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi); 1872 const struct intel_plane_state *old_plane_state; 1873 const struct intel_plane_state *new_plane_state; 1874 struct intel_plane *plane; 1875 enum plane_id plane_id; 1876 int level, ret, i; 1877 unsigned int dirty = 0; 1878 1879 for_each_oldnew_intel_plane_in_state(state, plane, 1880 old_plane_state, 1881 new_plane_state, i) { 1882 if (new_plane_state->hw.crtc != &crtc->base && 1883 old_plane_state->hw.crtc != &crtc->base) 1884 continue; 1885 1886 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state)) 1887 dirty |= BIT(plane->id); 1888 } 1889 1890 /* 1891 * DSPARB registers may have been reset due to the 1892 * power well being turned off. Make sure we restore 1893 * them to a consistent state even if no primary/sprite 1894 * planes are initially active. 1895 */ 1896 if (needs_modeset) 1897 crtc_state->fifo_changed = true; 1898 1899 if (!dirty) 1900 return 0; 1901 1902 /* cursor changes don't warrant a FIFO recompute */ 1903 if (dirty & ~BIT(PLANE_CURSOR)) { 1904 const struct intel_crtc_state *old_crtc_state = 1905 intel_atomic_get_old_crtc_state(state, crtc); 1906 const struct vlv_fifo_state *old_fifo_state = 1907 &old_crtc_state->wm.vlv.fifo_state; 1908 1909 ret = vlv_compute_fifo(crtc_state); 1910 if (ret) 1911 return ret; 1912 1913 if (needs_modeset || 1914 memcmp(old_fifo_state, fifo_state, 1915 sizeof(*fifo_state)) != 0) 1916 crtc_state->fifo_changed = true; 1917 } 1918 1919 /* initially allow all levels */ 1920 wm_state->num_levels = intel_wm_num_levels(dev_priv); 1921 /* 1922 * Note that enabling cxsr with no primary/sprite planes 1923 * enabled can wedge the pipe. Hence we only allow cxsr 1924 * with exactly one enabled primary/sprite plane. 1925 */ 1926 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1; 1927 1928 for (level = 0; level < wm_state->num_levels; level++) { 1929 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level]; 1930 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1; 1931 1932 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level)) 1933 break; 1934 1935 for_each_plane_id_on_crtc(crtc, plane_id) { 1936 wm_state->wm[level].plane[plane_id] = 1937 vlv_invert_wm_value(raw->plane[plane_id], 1938 fifo_state->plane[plane_id]); 1939 } 1940 1941 wm_state->sr[level].plane = 1942 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY], 1943 raw->plane[PLANE_SPRITE0], 1944 raw->plane[PLANE_SPRITE1]), 1945 sr_fifo_size); 1946 1947 wm_state->sr[level].cursor = 1948 vlv_invert_wm_value(raw->plane[PLANE_CURSOR], 1949 63); 1950 } 1951 1952 if (level == 0) 1953 return -EINVAL; 1954 1955 /* limit to only levels we can actually handle */ 1956 wm_state->num_levels = level; 1957 1958 /* invalidate the higher levels */ 1959 vlv_invalidate_wms(crtc, wm_state, level); 1960 1961 return 0; 1962 } 1963 1964 #define VLV_FIFO(plane, value) \ 1965 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV) 1966 1967 static void vlv_atomic_update_fifo(struct intel_atomic_state *state, 1968 struct intel_crtc *crtc) 1969 { 1970 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 1971 struct intel_uncore *uncore = &dev_priv->uncore; 1972 const struct intel_crtc_state *crtc_state = 1973 intel_atomic_get_new_crtc_state(state, crtc); 1974 const struct vlv_fifo_state *fifo_state = 1975 &crtc_state->wm.vlv.fifo_state; 1976 int sprite0_start, sprite1_start, fifo_size; 1977 1978 if (!crtc_state->fifo_changed) 1979 return; 1980 1981 sprite0_start = fifo_state->plane[PLANE_PRIMARY]; 1982 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start; 1983 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start; 1984 1985 WARN_ON(fifo_state->plane[PLANE_CURSOR] != 63); 1986 WARN_ON(fifo_size != 511); 1987 1988 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size); 1989 1990 /* 1991 * uncore.lock serves a double purpose here. It allows us to 1992 * use the less expensive I915_{READ,WRITE}_FW() functions, and 1993 * it protects the DSPARB registers from getting clobbered by 1994 * parallel updates from multiple pipes. 1995 * 1996 * intel_pipe_update_start() has already disabled interrupts 1997 * for us, so a plain spin_lock() is sufficient here. 1998 */ 1999 spin_lock(&uncore->lock); 2000 2001 switch (crtc->pipe) { 2002 u32 dsparb, dsparb2, dsparb3; 2003 case PIPE_A: 2004 dsparb = intel_uncore_read_fw(uncore, DSPARB); 2005 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2); 2006 2007 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) | 2008 VLV_FIFO(SPRITEB, 0xff)); 2009 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) | 2010 VLV_FIFO(SPRITEB, sprite1_start)); 2011 2012 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) | 2013 VLV_FIFO(SPRITEB_HI, 0x1)); 2014 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) | 2015 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8)); 2016 2017 intel_uncore_write_fw(uncore, DSPARB, dsparb); 2018 intel_uncore_write_fw(uncore, DSPARB2, dsparb2); 2019 break; 2020 case PIPE_B: 2021 dsparb = intel_uncore_read_fw(uncore, DSPARB); 2022 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2); 2023 2024 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) | 2025 VLV_FIFO(SPRITED, 0xff)); 2026 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) | 2027 VLV_FIFO(SPRITED, sprite1_start)); 2028 2029 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) | 2030 VLV_FIFO(SPRITED_HI, 0xff)); 2031 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) | 2032 VLV_FIFO(SPRITED_HI, sprite1_start >> 8)); 2033 2034 intel_uncore_write_fw(uncore, DSPARB, dsparb); 2035 intel_uncore_write_fw(uncore, DSPARB2, dsparb2); 2036 break; 2037 case PIPE_C: 2038 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3); 2039 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2); 2040 2041 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) | 2042 VLV_FIFO(SPRITEF, 0xff)); 2043 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) | 2044 VLV_FIFO(SPRITEF, sprite1_start)); 2045 2046 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) | 2047 VLV_FIFO(SPRITEF_HI, 0xff)); 2048 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) | 2049 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8)); 2050 2051 intel_uncore_write_fw(uncore, DSPARB3, dsparb3); 2052 intel_uncore_write_fw(uncore, DSPARB2, dsparb2); 2053 break; 2054 default: 2055 break; 2056 } 2057 2058 intel_uncore_posting_read_fw(uncore, DSPARB); 2059 2060 spin_unlock(&uncore->lock); 2061 } 2062 2063 #undef VLV_FIFO 2064 2065 static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state) 2066 { 2067 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); 2068 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate; 2069 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal; 2070 struct intel_atomic_state *intel_state = 2071 to_intel_atomic_state(new_crtc_state->uapi.state); 2072 const struct intel_crtc_state *old_crtc_state = 2073 intel_atomic_get_old_crtc_state(intel_state, crtc); 2074 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal; 2075 int level; 2076 2077 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) { 2078 *intermediate = *optimal; 2079 2080 intermediate->cxsr = false; 2081 goto out; 2082 } 2083 2084 intermediate->num_levels = min(optimal->num_levels, active->num_levels); 2085 intermediate->cxsr = optimal->cxsr && active->cxsr && 2086 !new_crtc_state->disable_cxsr; 2087 2088 for (level = 0; level < intermediate->num_levels; level++) { 2089 enum plane_id plane_id; 2090 2091 for_each_plane_id_on_crtc(crtc, plane_id) { 2092 intermediate->wm[level].plane[plane_id] = 2093 min(optimal->wm[level].plane[plane_id], 2094 active->wm[level].plane[plane_id]); 2095 } 2096 2097 intermediate->sr[level].plane = min(optimal->sr[level].plane, 2098 active->sr[level].plane); 2099 intermediate->sr[level].cursor = min(optimal->sr[level].cursor, 2100 active->sr[level].cursor); 2101 } 2102 2103 vlv_invalidate_wms(crtc, intermediate, level); 2104 2105 out: 2106 /* 2107 * If our intermediate WM are identical to the final WM, then we can 2108 * omit the post-vblank programming; only update if it's different. 2109 */ 2110 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0) 2111 new_crtc_state->wm.need_postvbl_update = true; 2112 2113 return 0; 2114 } 2115 2116 static void vlv_merge_wm(struct drm_i915_private *dev_priv, 2117 struct vlv_wm_values *wm) 2118 { 2119 struct intel_crtc *crtc; 2120 int num_active_pipes = 0; 2121 2122 wm->level = dev_priv->wm.max_level; 2123 wm->cxsr = true; 2124 2125 for_each_intel_crtc(&dev_priv->drm, crtc) { 2126 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv; 2127 2128 if (!crtc->active) 2129 continue; 2130 2131 if (!wm_state->cxsr) 2132 wm->cxsr = false; 2133 2134 num_active_pipes++; 2135 wm->level = min_t(int, wm->level, wm_state->num_levels - 1); 2136 } 2137 2138 if (num_active_pipes != 1) 2139 wm->cxsr = false; 2140 2141 if (num_active_pipes > 1) 2142 wm->level = VLV_WM_LEVEL_PM2; 2143 2144 for_each_intel_crtc(&dev_priv->drm, crtc) { 2145 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv; 2146 enum pipe pipe = crtc->pipe; 2147 2148 wm->pipe[pipe] = wm_state->wm[wm->level]; 2149 if (crtc->active && wm->cxsr) 2150 wm->sr = wm_state->sr[wm->level]; 2151 2152 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2; 2153 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2; 2154 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2; 2155 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2; 2156 } 2157 } 2158 2159 static void vlv_program_watermarks(struct drm_i915_private *dev_priv) 2160 { 2161 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv; 2162 struct vlv_wm_values new_wm = {}; 2163 2164 vlv_merge_wm(dev_priv, &new_wm); 2165 2166 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0) 2167 return; 2168 2169 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS)) 2170 chv_set_memory_dvfs(dev_priv, false); 2171 2172 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5)) 2173 chv_set_memory_pm5(dev_priv, false); 2174 2175 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true)) 2176 _intel_set_memory_cxsr(dev_priv, false); 2177 2178 vlv_write_wm_values(dev_priv, &new_wm); 2179 2180 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true)) 2181 _intel_set_memory_cxsr(dev_priv, true); 2182 2183 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5)) 2184 chv_set_memory_pm5(dev_priv, true); 2185 2186 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS)) 2187 chv_set_memory_dvfs(dev_priv, true); 2188 2189 *old_wm = new_wm; 2190 } 2191 2192 static void vlv_initial_watermarks(struct intel_atomic_state *state, 2193 struct intel_crtc *crtc) 2194 { 2195 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 2196 const struct intel_crtc_state *crtc_state = 2197 intel_atomic_get_new_crtc_state(state, crtc); 2198 2199 mutex_lock(&dev_priv->wm.wm_mutex); 2200 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate; 2201 vlv_program_watermarks(dev_priv); 2202 mutex_unlock(&dev_priv->wm.wm_mutex); 2203 } 2204 2205 static void vlv_optimize_watermarks(struct intel_atomic_state *state, 2206 struct intel_crtc *crtc) 2207 { 2208 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 2209 const struct intel_crtc_state *crtc_state = 2210 intel_atomic_get_new_crtc_state(state, crtc); 2211 2212 if (!crtc_state->wm.need_postvbl_update) 2213 return; 2214 2215 mutex_lock(&dev_priv->wm.wm_mutex); 2216 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal; 2217 vlv_program_watermarks(dev_priv); 2218 mutex_unlock(&dev_priv->wm.wm_mutex); 2219 } 2220 2221 static void i965_update_wm(struct intel_crtc *unused_crtc) 2222 { 2223 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); 2224 struct intel_crtc *crtc; 2225 int srwm = 1; 2226 int cursor_sr = 16; 2227 bool cxsr_enabled; 2228 2229 /* Calc sr entries for one plane configs */ 2230 crtc = single_enabled_crtc(dev_priv); 2231 if (crtc) { 2232 /* self-refresh has much higher latency */ 2233 static const int sr_latency_ns = 12000; 2234 const struct drm_display_mode *adjusted_mode = 2235 &crtc->config->hw.adjusted_mode; 2236 const struct drm_framebuffer *fb = 2237 crtc->base.primary->state->fb; 2238 int clock = adjusted_mode->crtc_clock; 2239 int htotal = adjusted_mode->crtc_htotal; 2240 int hdisplay = crtc->config->pipe_src_w; 2241 int cpp = fb->format->cpp[0]; 2242 int entries; 2243 2244 entries = intel_wm_method2(clock, htotal, 2245 hdisplay, cpp, sr_latency_ns / 100); 2246 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE); 2247 srwm = I965_FIFO_SIZE - entries; 2248 if (srwm < 0) 2249 srwm = 1; 2250 srwm &= 0x1ff; 2251 drm_dbg_kms(&dev_priv->drm, 2252 "self-refresh entries: %d, wm: %d\n", 2253 entries, srwm); 2254 2255 entries = intel_wm_method2(clock, htotal, 2256 crtc->base.cursor->state->crtc_w, 4, 2257 sr_latency_ns / 100); 2258 entries = DIV_ROUND_UP(entries, 2259 i965_cursor_wm_info.cacheline_size) + 2260 i965_cursor_wm_info.guard_size; 2261 2262 cursor_sr = i965_cursor_wm_info.fifo_size - entries; 2263 if (cursor_sr > i965_cursor_wm_info.max_wm) 2264 cursor_sr = i965_cursor_wm_info.max_wm; 2265 2266 drm_dbg_kms(&dev_priv->drm, 2267 "self-refresh watermark: display plane %d " 2268 "cursor %d\n", srwm, cursor_sr); 2269 2270 cxsr_enabled = true; 2271 } else { 2272 cxsr_enabled = false; 2273 /* Turn off self refresh if both pipes are enabled */ 2274 intel_set_memory_cxsr(dev_priv, false); 2275 } 2276 2277 drm_dbg_kms(&dev_priv->drm, 2278 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n", 2279 srwm); 2280 2281 /* 965 has limitations... */ 2282 I915_WRITE(DSPFW1, FW_WM(srwm, SR) | 2283 FW_WM(8, CURSORB) | 2284 FW_WM(8, PLANEB) | 2285 FW_WM(8, PLANEA)); 2286 I915_WRITE(DSPFW2, FW_WM(8, CURSORA) | 2287 FW_WM(8, PLANEC_OLD)); 2288 /* update cursor SR watermark */ 2289 I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR)); 2290 2291 if (cxsr_enabled) 2292 intel_set_memory_cxsr(dev_priv, true); 2293 } 2294 2295 #undef FW_WM 2296 2297 static void i9xx_update_wm(struct intel_crtc *unused_crtc) 2298 { 2299 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); 2300 const struct intel_watermark_params *wm_info; 2301 u32 fwater_lo; 2302 u32 fwater_hi; 2303 int cwm, srwm = 1; 2304 int fifo_size; 2305 int planea_wm, planeb_wm; 2306 struct intel_crtc *crtc, *enabled = NULL; 2307 2308 if (IS_I945GM(dev_priv)) 2309 wm_info = &i945_wm_info; 2310 else if (!IS_GEN(dev_priv, 2)) 2311 wm_info = &i915_wm_info; 2312 else 2313 wm_info = &i830_a_wm_info; 2314 2315 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A); 2316 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A); 2317 if (intel_crtc_active(crtc)) { 2318 const struct drm_display_mode *adjusted_mode = 2319 &crtc->config->hw.adjusted_mode; 2320 const struct drm_framebuffer *fb = 2321 crtc->base.primary->state->fb; 2322 int cpp; 2323 2324 if (IS_GEN(dev_priv, 2)) 2325 cpp = 4; 2326 else 2327 cpp = fb->format->cpp[0]; 2328 2329 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, 2330 wm_info, fifo_size, cpp, 2331 pessimal_latency_ns); 2332 enabled = crtc; 2333 } else { 2334 planea_wm = fifo_size - wm_info->guard_size; 2335 if (planea_wm > (long)wm_info->max_wm) 2336 planea_wm = wm_info->max_wm; 2337 } 2338 2339 if (IS_GEN(dev_priv, 2)) 2340 wm_info = &i830_bc_wm_info; 2341 2342 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B); 2343 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B); 2344 if (intel_crtc_active(crtc)) { 2345 const struct drm_display_mode *adjusted_mode = 2346 &crtc->config->hw.adjusted_mode; 2347 const struct drm_framebuffer *fb = 2348 crtc->base.primary->state->fb; 2349 int cpp; 2350 2351 if (IS_GEN(dev_priv, 2)) 2352 cpp = 4; 2353 else 2354 cpp = fb->format->cpp[0]; 2355 2356 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock, 2357 wm_info, fifo_size, cpp, 2358 pessimal_latency_ns); 2359 if (enabled == NULL) 2360 enabled = crtc; 2361 else 2362 enabled = NULL; 2363 } else { 2364 planeb_wm = fifo_size - wm_info->guard_size; 2365 if (planeb_wm > (long)wm_info->max_wm) 2366 planeb_wm = wm_info->max_wm; 2367 } 2368 2369 drm_dbg_kms(&dev_priv->drm, 2370 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm); 2371 2372 if (IS_I915GM(dev_priv) && enabled) { 2373 struct drm_i915_gem_object *obj; 2374 2375 obj = intel_fb_obj(enabled->base.primary->state->fb); 2376 2377 /* self-refresh seems busted with untiled */ 2378 if (!i915_gem_object_is_tiled(obj)) 2379 enabled = NULL; 2380 } 2381 2382 /* 2383 * Overlay gets an aggressive default since video jitter is bad. 2384 */ 2385 cwm = 2; 2386 2387 /* Play safe and disable self-refresh before adjusting watermarks. */ 2388 intel_set_memory_cxsr(dev_priv, false); 2389 2390 /* Calc sr entries for one plane configs */ 2391 if (HAS_FW_BLC(dev_priv) && enabled) { 2392 /* self-refresh has much higher latency */ 2393 static const int sr_latency_ns = 6000; 2394 const struct drm_display_mode *adjusted_mode = 2395 &enabled->config->hw.adjusted_mode; 2396 const struct drm_framebuffer *fb = 2397 enabled->base.primary->state->fb; 2398 int clock = adjusted_mode->crtc_clock; 2399 int htotal = adjusted_mode->crtc_htotal; 2400 int hdisplay = enabled->config->pipe_src_w; 2401 int cpp; 2402 int entries; 2403 2404 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv)) 2405 cpp = 4; 2406 else 2407 cpp = fb->format->cpp[0]; 2408 2409 entries = intel_wm_method2(clock, htotal, hdisplay, cpp, 2410 sr_latency_ns / 100); 2411 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size); 2412 drm_dbg_kms(&dev_priv->drm, 2413 "self-refresh entries: %d\n", entries); 2414 srwm = wm_info->fifo_size - entries; 2415 if (srwm < 0) 2416 srwm = 1; 2417 2418 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) 2419 I915_WRITE(FW_BLC_SELF, 2420 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff)); 2421 else 2422 I915_WRITE(FW_BLC_SELF, srwm & 0x3f); 2423 } 2424 2425 drm_dbg_kms(&dev_priv->drm, 2426 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n", 2427 planea_wm, planeb_wm, cwm, srwm); 2428 2429 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f); 2430 fwater_hi = (cwm & 0x1f); 2431 2432 /* Set request length to 8 cachelines per fetch */ 2433 fwater_lo = fwater_lo | (1 << 24) | (1 << 8); 2434 fwater_hi = fwater_hi | (1 << 8); 2435 2436 I915_WRITE(FW_BLC, fwater_lo); 2437 I915_WRITE(FW_BLC2, fwater_hi); 2438 2439 if (enabled) 2440 intel_set_memory_cxsr(dev_priv, true); 2441 } 2442 2443 static void i845_update_wm(struct intel_crtc *unused_crtc) 2444 { 2445 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); 2446 struct intel_crtc *crtc; 2447 const struct drm_display_mode *adjusted_mode; 2448 u32 fwater_lo; 2449 int planea_wm; 2450 2451 crtc = single_enabled_crtc(dev_priv); 2452 if (crtc == NULL) 2453 return; 2454 2455 adjusted_mode = &crtc->config->hw.adjusted_mode; 2456 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, 2457 &i845_wm_info, 2458 dev_priv->display.get_fifo_size(dev_priv, PLANE_A), 2459 4, pessimal_latency_ns); 2460 fwater_lo = I915_READ(FW_BLC) & ~0xfff; 2461 fwater_lo |= (3<<8) | planea_wm; 2462 2463 drm_dbg_kms(&dev_priv->drm, 2464 "Setting FIFO watermarks - A: %d\n", planea_wm); 2465 2466 I915_WRITE(FW_BLC, fwater_lo); 2467 } 2468 2469 /* latency must be in 0.1us units. */ 2470 static unsigned int ilk_wm_method1(unsigned int pixel_rate, 2471 unsigned int cpp, 2472 unsigned int latency) 2473 { 2474 unsigned int ret; 2475 2476 ret = intel_wm_method1(pixel_rate, cpp, latency); 2477 ret = DIV_ROUND_UP(ret, 64) + 2; 2478 2479 return ret; 2480 } 2481 2482 /* latency must be in 0.1us units. */ 2483 static unsigned int ilk_wm_method2(unsigned int pixel_rate, 2484 unsigned int htotal, 2485 unsigned int width, 2486 unsigned int cpp, 2487 unsigned int latency) 2488 { 2489 unsigned int ret; 2490 2491 ret = intel_wm_method2(pixel_rate, htotal, 2492 width, cpp, latency); 2493 ret = DIV_ROUND_UP(ret, 64) + 2; 2494 2495 return ret; 2496 } 2497 2498 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp) 2499 { 2500 /* 2501 * Neither of these should be possible since this function shouldn't be 2502 * called if the CRTC is off or the plane is invisible. But let's be 2503 * extra paranoid to avoid a potential divide-by-zero if we screw up 2504 * elsewhere in the driver. 2505 */ 2506 if (WARN_ON(!cpp)) 2507 return 0; 2508 if (WARN_ON(!horiz_pixels)) 2509 return 0; 2510 2511 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2; 2512 } 2513 2514 struct ilk_wm_maximums { 2515 u16 pri; 2516 u16 spr; 2517 u16 cur; 2518 u16 fbc; 2519 }; 2520 2521 /* 2522 * For both WM_PIPE and WM_LP. 2523 * mem_value must be in 0.1us units. 2524 */ 2525 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state, 2526 const struct intel_plane_state *plane_state, 2527 u32 mem_value, bool is_lp) 2528 { 2529 u32 method1, method2; 2530 int cpp; 2531 2532 if (mem_value == 0) 2533 return U32_MAX; 2534 2535 if (!intel_wm_plane_visible(crtc_state, plane_state)) 2536 return 0; 2537 2538 cpp = plane_state->hw.fb->format->cpp[0]; 2539 2540 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value); 2541 2542 if (!is_lp) 2543 return method1; 2544 2545 method2 = ilk_wm_method2(crtc_state->pixel_rate, 2546 crtc_state->hw.adjusted_mode.crtc_htotal, 2547 drm_rect_width(&plane_state->uapi.dst), 2548 cpp, mem_value); 2549 2550 return min(method1, method2); 2551 } 2552 2553 /* 2554 * For both WM_PIPE and WM_LP. 2555 * mem_value must be in 0.1us units. 2556 */ 2557 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state, 2558 const struct intel_plane_state *plane_state, 2559 u32 mem_value) 2560 { 2561 u32 method1, method2; 2562 int cpp; 2563 2564 if (mem_value == 0) 2565 return U32_MAX; 2566 2567 if (!intel_wm_plane_visible(crtc_state, plane_state)) 2568 return 0; 2569 2570 cpp = plane_state->hw.fb->format->cpp[0]; 2571 2572 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value); 2573 method2 = ilk_wm_method2(crtc_state->pixel_rate, 2574 crtc_state->hw.adjusted_mode.crtc_htotal, 2575 drm_rect_width(&plane_state->uapi.dst), 2576 cpp, mem_value); 2577 return min(method1, method2); 2578 } 2579 2580 /* 2581 * For both WM_PIPE and WM_LP. 2582 * mem_value must be in 0.1us units. 2583 */ 2584 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state, 2585 const struct intel_plane_state *plane_state, 2586 u32 mem_value) 2587 { 2588 int cpp; 2589 2590 if (mem_value == 0) 2591 return U32_MAX; 2592 2593 if (!intel_wm_plane_visible(crtc_state, plane_state)) 2594 return 0; 2595 2596 cpp = plane_state->hw.fb->format->cpp[0]; 2597 2598 return ilk_wm_method2(crtc_state->pixel_rate, 2599 crtc_state->hw.adjusted_mode.crtc_htotal, 2600 drm_rect_width(&plane_state->uapi.dst), 2601 cpp, mem_value); 2602 } 2603 2604 /* Only for WM_LP. */ 2605 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state, 2606 const struct intel_plane_state *plane_state, 2607 u32 pri_val) 2608 { 2609 int cpp; 2610 2611 if (!intel_wm_plane_visible(crtc_state, plane_state)) 2612 return 0; 2613 2614 cpp = plane_state->hw.fb->format->cpp[0]; 2615 2616 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst), 2617 cpp); 2618 } 2619 2620 static unsigned int 2621 ilk_display_fifo_size(const struct drm_i915_private *dev_priv) 2622 { 2623 if (INTEL_GEN(dev_priv) >= 8) 2624 return 3072; 2625 else if (INTEL_GEN(dev_priv) >= 7) 2626 return 768; 2627 else 2628 return 512; 2629 } 2630 2631 static unsigned int 2632 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv, 2633 int level, bool is_sprite) 2634 { 2635 if (INTEL_GEN(dev_priv) >= 8) 2636 /* BDW primary/sprite plane watermarks */ 2637 return level == 0 ? 255 : 2047; 2638 else if (INTEL_GEN(dev_priv) >= 7) 2639 /* IVB/HSW primary/sprite plane watermarks */ 2640 return level == 0 ? 127 : 1023; 2641 else if (!is_sprite) 2642 /* ILK/SNB primary plane watermarks */ 2643 return level == 0 ? 127 : 511; 2644 else 2645 /* ILK/SNB sprite plane watermarks */ 2646 return level == 0 ? 63 : 255; 2647 } 2648 2649 static unsigned int 2650 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level) 2651 { 2652 if (INTEL_GEN(dev_priv) >= 7) 2653 return level == 0 ? 63 : 255; 2654 else 2655 return level == 0 ? 31 : 63; 2656 } 2657 2658 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv) 2659 { 2660 if (INTEL_GEN(dev_priv) >= 8) 2661 return 31; 2662 else 2663 return 15; 2664 } 2665 2666 /* Calculate the maximum primary/sprite plane watermark */ 2667 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv, 2668 int level, 2669 const struct intel_wm_config *config, 2670 enum intel_ddb_partitioning ddb_partitioning, 2671 bool is_sprite) 2672 { 2673 unsigned int fifo_size = ilk_display_fifo_size(dev_priv); 2674 2675 /* if sprites aren't enabled, sprites get nothing */ 2676 if (is_sprite && !config->sprites_enabled) 2677 return 0; 2678 2679 /* HSW allows LP1+ watermarks even with multiple pipes */ 2680 if (level == 0 || config->num_pipes_active > 1) { 2681 fifo_size /= INTEL_NUM_PIPES(dev_priv); 2682 2683 /* 2684 * For some reason the non self refresh 2685 * FIFO size is only half of the self 2686 * refresh FIFO size on ILK/SNB. 2687 */ 2688 if (INTEL_GEN(dev_priv) <= 6) 2689 fifo_size /= 2; 2690 } 2691 2692 if (config->sprites_enabled) { 2693 /* level 0 is always calculated with 1:1 split */ 2694 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) { 2695 if (is_sprite) 2696 fifo_size *= 5; 2697 fifo_size /= 6; 2698 } else { 2699 fifo_size /= 2; 2700 } 2701 } 2702 2703 /* clamp to max that the registers can hold */ 2704 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite)); 2705 } 2706 2707 /* Calculate the maximum cursor plane watermark */ 2708 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv, 2709 int level, 2710 const struct intel_wm_config *config) 2711 { 2712 /* HSW LP1+ watermarks w/ multiple pipes */ 2713 if (level > 0 && config->num_pipes_active > 1) 2714 return 64; 2715 2716 /* otherwise just report max that registers can hold */ 2717 return ilk_cursor_wm_reg_max(dev_priv, level); 2718 } 2719 2720 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv, 2721 int level, 2722 const struct intel_wm_config *config, 2723 enum intel_ddb_partitioning ddb_partitioning, 2724 struct ilk_wm_maximums *max) 2725 { 2726 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false); 2727 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true); 2728 max->cur = ilk_cursor_wm_max(dev_priv, level, config); 2729 max->fbc = ilk_fbc_wm_reg_max(dev_priv); 2730 } 2731 2732 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv, 2733 int level, 2734 struct ilk_wm_maximums *max) 2735 { 2736 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false); 2737 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true); 2738 max->cur = ilk_cursor_wm_reg_max(dev_priv, level); 2739 max->fbc = ilk_fbc_wm_reg_max(dev_priv); 2740 } 2741 2742 static bool ilk_validate_wm_level(int level, 2743 const struct ilk_wm_maximums *max, 2744 struct intel_wm_level *result) 2745 { 2746 bool ret; 2747 2748 /* already determined to be invalid? */ 2749 if (!result->enable) 2750 return false; 2751 2752 result->enable = result->pri_val <= max->pri && 2753 result->spr_val <= max->spr && 2754 result->cur_val <= max->cur; 2755 2756 ret = result->enable; 2757 2758 /* 2759 * HACK until we can pre-compute everything, 2760 * and thus fail gracefully if LP0 watermarks 2761 * are exceeded... 2762 */ 2763 if (level == 0 && !result->enable) { 2764 if (result->pri_val > max->pri) 2765 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n", 2766 level, result->pri_val, max->pri); 2767 if (result->spr_val > max->spr) 2768 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n", 2769 level, result->spr_val, max->spr); 2770 if (result->cur_val > max->cur) 2771 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n", 2772 level, result->cur_val, max->cur); 2773 2774 result->pri_val = min_t(u32, result->pri_val, max->pri); 2775 result->spr_val = min_t(u32, result->spr_val, max->spr); 2776 result->cur_val = min_t(u32, result->cur_val, max->cur); 2777 result->enable = true; 2778 } 2779 2780 return ret; 2781 } 2782 2783 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, 2784 const struct intel_crtc *intel_crtc, 2785 int level, 2786 struct intel_crtc_state *crtc_state, 2787 const struct intel_plane_state *pristate, 2788 const struct intel_plane_state *sprstate, 2789 const struct intel_plane_state *curstate, 2790 struct intel_wm_level *result) 2791 { 2792 u16 pri_latency = dev_priv->wm.pri_latency[level]; 2793 u16 spr_latency = dev_priv->wm.spr_latency[level]; 2794 u16 cur_latency = dev_priv->wm.cur_latency[level]; 2795 2796 /* WM1+ latency values stored in 0.5us units */ 2797 if (level > 0) { 2798 pri_latency *= 5; 2799 spr_latency *= 5; 2800 cur_latency *= 5; 2801 } 2802 2803 if (pristate) { 2804 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate, 2805 pri_latency, level); 2806 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val); 2807 } 2808 2809 if (sprstate) 2810 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency); 2811 2812 if (curstate) 2813 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency); 2814 2815 result->enable = true; 2816 } 2817 2818 static u32 2819 hsw_compute_linetime_wm(const struct intel_crtc_state *crtc_state) 2820 { 2821 const struct intel_atomic_state *intel_state = 2822 to_intel_atomic_state(crtc_state->uapi.state); 2823 const struct drm_display_mode *adjusted_mode = 2824 &crtc_state->hw.adjusted_mode; 2825 u32 linetime, ips_linetime; 2826 2827 if (!crtc_state->hw.active) 2828 return 0; 2829 if (WARN_ON(adjusted_mode->crtc_clock == 0)) 2830 return 0; 2831 if (WARN_ON(intel_state->cdclk.logical.cdclk == 0)) 2832 return 0; 2833 2834 /* The WM are computed with base on how long it takes to fill a single 2835 * row at the given clock rate, multiplied by 8. 2836 * */ 2837 linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8, 2838 adjusted_mode->crtc_clock); 2839 ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8, 2840 intel_state->cdclk.logical.cdclk); 2841 2842 return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) | 2843 PIPE_WM_LINETIME_TIME(linetime); 2844 } 2845 2846 static void intel_read_wm_latency(struct drm_i915_private *dev_priv, 2847 u16 wm[8]) 2848 { 2849 struct intel_uncore *uncore = &dev_priv->uncore; 2850 2851 if (INTEL_GEN(dev_priv) >= 9) { 2852 u32 val; 2853 int ret, i; 2854 int level, max_level = ilk_wm_max_level(dev_priv); 2855 2856 /* read the first set of memory latencies[0:3] */ 2857 val = 0; /* data0 to be programmed to 0 for first set */ 2858 ret = sandybridge_pcode_read(dev_priv, 2859 GEN9_PCODE_READ_MEM_LATENCY, 2860 &val, NULL); 2861 2862 if (ret) { 2863 drm_err(&dev_priv->drm, 2864 "SKL Mailbox read error = %d\n", ret); 2865 return; 2866 } 2867 2868 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK; 2869 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) & 2870 GEN9_MEM_LATENCY_LEVEL_MASK; 2871 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) & 2872 GEN9_MEM_LATENCY_LEVEL_MASK; 2873 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) & 2874 GEN9_MEM_LATENCY_LEVEL_MASK; 2875 2876 /* read the second set of memory latencies[4:7] */ 2877 val = 1; /* data0 to be programmed to 1 for second set */ 2878 ret = sandybridge_pcode_read(dev_priv, 2879 GEN9_PCODE_READ_MEM_LATENCY, 2880 &val, NULL); 2881 if (ret) { 2882 drm_err(&dev_priv->drm, 2883 "SKL Mailbox read error = %d\n", ret); 2884 return; 2885 } 2886 2887 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK; 2888 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) & 2889 GEN9_MEM_LATENCY_LEVEL_MASK; 2890 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) & 2891 GEN9_MEM_LATENCY_LEVEL_MASK; 2892 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) & 2893 GEN9_MEM_LATENCY_LEVEL_MASK; 2894 2895 /* 2896 * If a level n (n > 1) has a 0us latency, all levels m (m >= n) 2897 * need to be disabled. We make sure to sanitize the values out 2898 * of the punit to satisfy this requirement. 2899 */ 2900 for (level = 1; level <= max_level; level++) { 2901 if (wm[level] == 0) { 2902 for (i = level + 1; i <= max_level; i++) 2903 wm[i] = 0; 2904 break; 2905 } 2906 } 2907 2908 /* 2909 * WaWmMemoryReadLatency:skl+,glk 2910 * 2911 * punit doesn't take into account the read latency so we need 2912 * to add 2us to the various latency levels we retrieve from the 2913 * punit when level 0 response data us 0us. 2914 */ 2915 if (wm[0] == 0) { 2916 wm[0] += 2; 2917 for (level = 1; level <= max_level; level++) { 2918 if (wm[level] == 0) 2919 break; 2920 wm[level] += 2; 2921 } 2922 } 2923 2924 /* 2925 * WA Level-0 adjustment for 16GB DIMMs: SKL+ 2926 * If we could not get dimm info enable this WA to prevent from 2927 * any underrun. If not able to get Dimm info assume 16GB dimm 2928 * to avoid any underrun. 2929 */ 2930 if (dev_priv->dram_info.is_16gb_dimm) 2931 wm[0] += 1; 2932 2933 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { 2934 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD); 2935 2936 wm[0] = (sskpd >> 56) & 0xFF; 2937 if (wm[0] == 0) 2938 wm[0] = sskpd & 0xF; 2939 wm[1] = (sskpd >> 4) & 0xFF; 2940 wm[2] = (sskpd >> 12) & 0xFF; 2941 wm[3] = (sskpd >> 20) & 0x1FF; 2942 wm[4] = (sskpd >> 32) & 0x1FF; 2943 } else if (INTEL_GEN(dev_priv) >= 6) { 2944 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD); 2945 2946 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK; 2947 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK; 2948 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK; 2949 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK; 2950 } else if (INTEL_GEN(dev_priv) >= 5) { 2951 u32 mltr = intel_uncore_read(uncore, MLTR_ILK); 2952 2953 /* ILK primary LP0 latency is 700 ns */ 2954 wm[0] = 7; 2955 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK; 2956 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK; 2957 } else { 2958 MISSING_CASE(INTEL_DEVID(dev_priv)); 2959 } 2960 } 2961 2962 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv, 2963 u16 wm[5]) 2964 { 2965 /* ILK sprite LP0 latency is 1300 ns */ 2966 if (IS_GEN(dev_priv, 5)) 2967 wm[0] = 13; 2968 } 2969 2970 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv, 2971 u16 wm[5]) 2972 { 2973 /* ILK cursor LP0 latency is 1300 ns */ 2974 if (IS_GEN(dev_priv, 5)) 2975 wm[0] = 13; 2976 } 2977 2978 int ilk_wm_max_level(const struct drm_i915_private *dev_priv) 2979 { 2980 /* how many WM levels are we expecting */ 2981 if (INTEL_GEN(dev_priv) >= 9) 2982 return 7; 2983 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) 2984 return 4; 2985 else if (INTEL_GEN(dev_priv) >= 6) 2986 return 3; 2987 else 2988 return 2; 2989 } 2990 2991 static void intel_print_wm_latency(struct drm_i915_private *dev_priv, 2992 const char *name, 2993 const u16 wm[8]) 2994 { 2995 int level, max_level = ilk_wm_max_level(dev_priv); 2996 2997 for (level = 0; level <= max_level; level++) { 2998 unsigned int latency = wm[level]; 2999 3000 if (latency == 0) { 3001 drm_dbg_kms(&dev_priv->drm, 3002 "%s WM%d latency not provided\n", 3003 name, level); 3004 continue; 3005 } 3006 3007 /* 3008 * - latencies are in us on gen9. 3009 * - before then, WM1+ latency values are in 0.5us units 3010 */ 3011 if (INTEL_GEN(dev_priv) >= 9) 3012 latency *= 10; 3013 else if (level > 0) 3014 latency *= 5; 3015 3016 drm_dbg_kms(&dev_priv->drm, 3017 "%s WM%d latency %u (%u.%u usec)\n", name, level, 3018 wm[level], latency / 10, latency % 10); 3019 } 3020 } 3021 3022 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv, 3023 u16 wm[5], u16 min) 3024 { 3025 int level, max_level = ilk_wm_max_level(dev_priv); 3026 3027 if (wm[0] >= min) 3028 return false; 3029 3030 wm[0] = max(wm[0], min); 3031 for (level = 1; level <= max_level; level++) 3032 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5)); 3033 3034 return true; 3035 } 3036 3037 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv) 3038 { 3039 bool changed; 3040 3041 /* 3042 * The BIOS provided WM memory latency values are often 3043 * inadequate for high resolution displays. Adjust them. 3044 */ 3045 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) | 3046 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) | 3047 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12); 3048 3049 if (!changed) 3050 return; 3051 3052 drm_dbg_kms(&dev_priv->drm, 3053 "WM latency values increased to avoid potential underruns\n"); 3054 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency); 3055 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency); 3056 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency); 3057 } 3058 3059 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv) 3060 { 3061 /* 3062 * On some SNB machines (Thinkpad X220 Tablet at least) 3063 * LP3 usage can cause vblank interrupts to be lost. 3064 * The DEIIR bit will go high but it looks like the CPU 3065 * never gets interrupted. 3066 * 3067 * It's not clear whether other interrupt source could 3068 * be affected or if this is somehow limited to vblank 3069 * interrupts only. To play it safe we disable LP3 3070 * watermarks entirely. 3071 */ 3072 if (dev_priv->wm.pri_latency[3] == 0 && 3073 dev_priv->wm.spr_latency[3] == 0 && 3074 dev_priv->wm.cur_latency[3] == 0) 3075 return; 3076 3077 dev_priv->wm.pri_latency[3] = 0; 3078 dev_priv->wm.spr_latency[3] = 0; 3079 dev_priv->wm.cur_latency[3] = 0; 3080 3081 drm_dbg_kms(&dev_priv->drm, 3082 "LP3 watermarks disabled due to potential for lost interrupts\n"); 3083 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency); 3084 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency); 3085 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency); 3086 } 3087 3088 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv) 3089 { 3090 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency); 3091 3092 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency, 3093 sizeof(dev_priv->wm.pri_latency)); 3094 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency, 3095 sizeof(dev_priv->wm.pri_latency)); 3096 3097 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency); 3098 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency); 3099 3100 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency); 3101 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency); 3102 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency); 3103 3104 if (IS_GEN(dev_priv, 6)) { 3105 snb_wm_latency_quirk(dev_priv); 3106 snb_wm_lp3_irq_quirk(dev_priv); 3107 } 3108 } 3109 3110 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv) 3111 { 3112 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency); 3113 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency); 3114 } 3115 3116 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv, 3117 struct intel_pipe_wm *pipe_wm) 3118 { 3119 /* LP0 watermark maximums depend on this pipe alone */ 3120 const struct intel_wm_config config = { 3121 .num_pipes_active = 1, 3122 .sprites_enabled = pipe_wm->sprites_enabled, 3123 .sprites_scaled = pipe_wm->sprites_scaled, 3124 }; 3125 struct ilk_wm_maximums max; 3126 3127 /* LP0 watermarks always use 1/2 DDB partitioning */ 3128 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max); 3129 3130 /* At least LP0 must be valid */ 3131 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) { 3132 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n"); 3133 return false; 3134 } 3135 3136 return true; 3137 } 3138 3139 /* Compute new watermarks for the pipe */ 3140 static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state) 3141 { 3142 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 3143 struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->uapi.crtc); 3144 struct intel_pipe_wm *pipe_wm; 3145 struct intel_plane *plane; 3146 const struct intel_plane_state *plane_state; 3147 const struct intel_plane_state *pristate = NULL; 3148 const struct intel_plane_state *sprstate = NULL; 3149 const struct intel_plane_state *curstate = NULL; 3150 int level, max_level = ilk_wm_max_level(dev_priv), usable_level; 3151 struct ilk_wm_maximums max; 3152 3153 pipe_wm = &crtc_state->wm.ilk.optimal; 3154 3155 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) { 3156 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY) 3157 pristate = plane_state; 3158 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY) 3159 sprstate = plane_state; 3160 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR) 3161 curstate = plane_state; 3162 } 3163 3164 pipe_wm->pipe_enabled = crtc_state->hw.active; 3165 if (sprstate) { 3166 pipe_wm->sprites_enabled = sprstate->uapi.visible; 3167 pipe_wm->sprites_scaled = sprstate->uapi.visible && 3168 (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 || 3169 drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16); 3170 } 3171 3172 usable_level = max_level; 3173 3174 /* ILK/SNB: LP2+ watermarks only w/o sprites */ 3175 if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled) 3176 usable_level = 1; 3177 3178 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */ 3179 if (pipe_wm->sprites_scaled) 3180 usable_level = 0; 3181 3182 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm)); 3183 ilk_compute_wm_level(dev_priv, intel_crtc, 0, crtc_state, 3184 pristate, sprstate, curstate, &pipe_wm->wm[0]); 3185 3186 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) 3187 pipe_wm->linetime = hsw_compute_linetime_wm(crtc_state); 3188 3189 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm)) 3190 return -EINVAL; 3191 3192 ilk_compute_wm_reg_maximums(dev_priv, 1, &max); 3193 3194 for (level = 1; level <= usable_level; level++) { 3195 struct intel_wm_level *wm = &pipe_wm->wm[level]; 3196 3197 ilk_compute_wm_level(dev_priv, intel_crtc, level, crtc_state, 3198 pristate, sprstate, curstate, wm); 3199 3200 /* 3201 * Disable any watermark level that exceeds the 3202 * register maximums since such watermarks are 3203 * always invalid. 3204 */ 3205 if (!ilk_validate_wm_level(level, &max, wm)) { 3206 memset(wm, 0, sizeof(*wm)); 3207 break; 3208 } 3209 } 3210 3211 return 0; 3212 } 3213 3214 /* 3215 * Build a set of 'intermediate' watermark values that satisfy both the old 3216 * state and the new state. These can be programmed to the hardware 3217 * immediately. 3218 */ 3219 static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate) 3220 { 3221 struct intel_crtc *intel_crtc = to_intel_crtc(newstate->uapi.crtc); 3222 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); 3223 struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate; 3224 struct intel_atomic_state *intel_state = 3225 to_intel_atomic_state(newstate->uapi.state); 3226 const struct intel_crtc_state *oldstate = 3227 intel_atomic_get_old_crtc_state(intel_state, intel_crtc); 3228 const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal; 3229 int level, max_level = ilk_wm_max_level(dev_priv); 3230 3231 /* 3232 * Start with the final, target watermarks, then combine with the 3233 * currently active watermarks to get values that are safe both before 3234 * and after the vblank. 3235 */ 3236 *a = newstate->wm.ilk.optimal; 3237 if (!newstate->hw.active || drm_atomic_crtc_needs_modeset(&newstate->uapi) || 3238 intel_state->skip_intermediate_wm) 3239 return 0; 3240 3241 a->pipe_enabled |= b->pipe_enabled; 3242 a->sprites_enabled |= b->sprites_enabled; 3243 a->sprites_scaled |= b->sprites_scaled; 3244 3245 for (level = 0; level <= max_level; level++) { 3246 struct intel_wm_level *a_wm = &a->wm[level]; 3247 const struct intel_wm_level *b_wm = &b->wm[level]; 3248 3249 a_wm->enable &= b_wm->enable; 3250 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val); 3251 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val); 3252 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val); 3253 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val); 3254 } 3255 3256 /* 3257 * We need to make sure that these merged watermark values are 3258 * actually a valid configuration themselves. If they're not, 3259 * there's no safe way to transition from the old state to 3260 * the new state, so we need to fail the atomic transaction. 3261 */ 3262 if (!ilk_validate_pipe_wm(dev_priv, a)) 3263 return -EINVAL; 3264 3265 /* 3266 * If our intermediate WM are identical to the final WM, then we can 3267 * omit the post-vblank programming; only update if it's different. 3268 */ 3269 if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0) 3270 newstate->wm.need_postvbl_update = true; 3271 3272 return 0; 3273 } 3274 3275 /* 3276 * Merge the watermarks from all active pipes for a specific level. 3277 */ 3278 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv, 3279 int level, 3280 struct intel_wm_level *ret_wm) 3281 { 3282 const struct intel_crtc *intel_crtc; 3283 3284 ret_wm->enable = true; 3285 3286 for_each_intel_crtc(&dev_priv->drm, intel_crtc) { 3287 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk; 3288 const struct intel_wm_level *wm = &active->wm[level]; 3289 3290 if (!active->pipe_enabled) 3291 continue; 3292 3293 /* 3294 * The watermark values may have been used in the past, 3295 * so we must maintain them in the registers for some 3296 * time even if the level is now disabled. 3297 */ 3298 if (!wm->enable) 3299 ret_wm->enable = false; 3300 3301 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val); 3302 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val); 3303 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val); 3304 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val); 3305 } 3306 } 3307 3308 /* 3309 * Merge all low power watermarks for all active pipes. 3310 */ 3311 static void ilk_wm_merge(struct drm_i915_private *dev_priv, 3312 const struct intel_wm_config *config, 3313 const struct ilk_wm_maximums *max, 3314 struct intel_pipe_wm *merged) 3315 { 3316 int level, max_level = ilk_wm_max_level(dev_priv); 3317 int last_enabled_level = max_level; 3318 3319 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */ 3320 if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) && 3321 config->num_pipes_active > 1) 3322 last_enabled_level = 0; 3323 3324 /* ILK: FBC WM must be disabled always */ 3325 merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6; 3326 3327 /* merge each WM1+ level */ 3328 for (level = 1; level <= max_level; level++) { 3329 struct intel_wm_level *wm = &merged->wm[level]; 3330 3331 ilk_merge_wm_level(dev_priv, level, wm); 3332 3333 if (level > last_enabled_level) 3334 wm->enable = false; 3335 else if (!ilk_validate_wm_level(level, max, wm)) 3336 /* make sure all following levels get disabled */ 3337 last_enabled_level = level - 1; 3338 3339 /* 3340 * The spec says it is preferred to disable 3341 * FBC WMs instead of disabling a WM level. 3342 */ 3343 if (wm->fbc_val > max->fbc) { 3344 if (wm->enable) 3345 merged->fbc_wm_enabled = false; 3346 wm->fbc_val = 0; 3347 } 3348 } 3349 3350 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */ 3351 /* 3352 * FIXME this is racy. FBC might get enabled later. 3353 * What we should check here is whether FBC can be 3354 * enabled sometime later. 3355 */ 3356 if (IS_GEN(dev_priv, 5) && !merged->fbc_wm_enabled && 3357 intel_fbc_is_active(dev_priv)) { 3358 for (level = 2; level <= max_level; level++) { 3359 struct intel_wm_level *wm = &merged->wm[level]; 3360 3361 wm->enable = false; 3362 } 3363 } 3364 } 3365 3366 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm) 3367 { 3368 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */ 3369 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable); 3370 } 3371 3372 /* The value we need to program into the WM_LPx latency field */ 3373 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv, 3374 int level) 3375 { 3376 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) 3377 return 2 * level; 3378 else 3379 return dev_priv->wm.pri_latency[level]; 3380 } 3381 3382 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv, 3383 const struct intel_pipe_wm *merged, 3384 enum intel_ddb_partitioning partitioning, 3385 struct ilk_wm_values *results) 3386 { 3387 struct intel_crtc *intel_crtc; 3388 int level, wm_lp; 3389 3390 results->enable_fbc_wm = merged->fbc_wm_enabled; 3391 results->partitioning = partitioning; 3392 3393 /* LP1+ register values */ 3394 for (wm_lp = 1; wm_lp <= 3; wm_lp++) { 3395 const struct intel_wm_level *r; 3396 3397 level = ilk_wm_lp_to_level(wm_lp, merged); 3398 3399 r = &merged->wm[level]; 3400 3401 /* 3402 * Maintain the watermark values even if the level is 3403 * disabled. Doing otherwise could cause underruns. 3404 */ 3405 results->wm_lp[wm_lp - 1] = 3406 (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) | 3407 (r->pri_val << WM1_LP_SR_SHIFT) | 3408 r->cur_val; 3409 3410 if (r->enable) 3411 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN; 3412 3413 if (INTEL_GEN(dev_priv) >= 8) 3414 results->wm_lp[wm_lp - 1] |= 3415 r->fbc_val << WM1_LP_FBC_SHIFT_BDW; 3416 else 3417 results->wm_lp[wm_lp - 1] |= 3418 r->fbc_val << WM1_LP_FBC_SHIFT; 3419 3420 /* 3421 * Always set WM1S_LP_EN when spr_val != 0, even if the 3422 * level is disabled. Doing otherwise could cause underruns. 3423 */ 3424 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) { 3425 WARN_ON(wm_lp != 1); 3426 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val; 3427 } else 3428 results->wm_lp_spr[wm_lp - 1] = r->spr_val; 3429 } 3430 3431 /* LP0 register values */ 3432 for_each_intel_crtc(&dev_priv->drm, intel_crtc) { 3433 enum pipe pipe = intel_crtc->pipe; 3434 const struct intel_wm_level *r = 3435 &intel_crtc->wm.active.ilk.wm[0]; 3436 3437 if (WARN_ON(!r->enable)) 3438 continue; 3439 3440 results->wm_linetime[pipe] = intel_crtc->wm.active.ilk.linetime; 3441 3442 results->wm_pipe[pipe] = 3443 (r->pri_val << WM0_PIPE_PLANE_SHIFT) | 3444 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) | 3445 r->cur_val; 3446 } 3447 } 3448 3449 /* Find the result with the highest level enabled. Check for enable_fbc_wm in 3450 * case both are at the same level. Prefer r1 in case they're the same. */ 3451 static struct intel_pipe_wm * 3452 ilk_find_best_result(struct drm_i915_private *dev_priv, 3453 struct intel_pipe_wm *r1, 3454 struct intel_pipe_wm *r2) 3455 { 3456 int level, max_level = ilk_wm_max_level(dev_priv); 3457 int level1 = 0, level2 = 0; 3458 3459 for (level = 1; level <= max_level; level++) { 3460 if (r1->wm[level].enable) 3461 level1 = level; 3462 if (r2->wm[level].enable) 3463 level2 = level; 3464 } 3465 3466 if (level1 == level2) { 3467 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled) 3468 return r2; 3469 else 3470 return r1; 3471 } else if (level1 > level2) { 3472 return r1; 3473 } else { 3474 return r2; 3475 } 3476 } 3477 3478 /* dirty bits used to track which watermarks need changes */ 3479 #define WM_DIRTY_PIPE(pipe) (1 << (pipe)) 3480 #define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe))) 3481 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp))) 3482 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3)) 3483 #define WM_DIRTY_FBC (1 << 24) 3484 #define WM_DIRTY_DDB (1 << 25) 3485 3486 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv, 3487 const struct ilk_wm_values *old, 3488 const struct ilk_wm_values *new) 3489 { 3490 unsigned int dirty = 0; 3491 enum pipe pipe; 3492 int wm_lp; 3493 3494 for_each_pipe(dev_priv, pipe) { 3495 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) { 3496 dirty |= WM_DIRTY_LINETIME(pipe); 3497 /* Must disable LP1+ watermarks too */ 3498 dirty |= WM_DIRTY_LP_ALL; 3499 } 3500 3501 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) { 3502 dirty |= WM_DIRTY_PIPE(pipe); 3503 /* Must disable LP1+ watermarks too */ 3504 dirty |= WM_DIRTY_LP_ALL; 3505 } 3506 } 3507 3508 if (old->enable_fbc_wm != new->enable_fbc_wm) { 3509 dirty |= WM_DIRTY_FBC; 3510 /* Must disable LP1+ watermarks too */ 3511 dirty |= WM_DIRTY_LP_ALL; 3512 } 3513 3514 if (old->partitioning != new->partitioning) { 3515 dirty |= WM_DIRTY_DDB; 3516 /* Must disable LP1+ watermarks too */ 3517 dirty |= WM_DIRTY_LP_ALL; 3518 } 3519 3520 /* LP1+ watermarks already deemed dirty, no need to continue */ 3521 if (dirty & WM_DIRTY_LP_ALL) 3522 return dirty; 3523 3524 /* Find the lowest numbered LP1+ watermark in need of an update... */ 3525 for (wm_lp = 1; wm_lp <= 3; wm_lp++) { 3526 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] || 3527 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1]) 3528 break; 3529 } 3530 3531 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */ 3532 for (; wm_lp <= 3; wm_lp++) 3533 dirty |= WM_DIRTY_LP(wm_lp); 3534 3535 return dirty; 3536 } 3537 3538 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv, 3539 unsigned int dirty) 3540 { 3541 struct ilk_wm_values *previous = &dev_priv->wm.hw; 3542 bool changed = false; 3543 3544 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) { 3545 previous->wm_lp[2] &= ~WM1_LP_SR_EN; 3546 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]); 3547 changed = true; 3548 } 3549 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) { 3550 previous->wm_lp[1] &= ~WM1_LP_SR_EN; 3551 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]); 3552 changed = true; 3553 } 3554 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) { 3555 previous->wm_lp[0] &= ~WM1_LP_SR_EN; 3556 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]); 3557 changed = true; 3558 } 3559 3560 /* 3561 * Don't touch WM1S_LP_EN here. 3562 * Doing so could cause underruns. 3563 */ 3564 3565 return changed; 3566 } 3567 3568 /* 3569 * The spec says we shouldn't write when we don't need, because every write 3570 * causes WMs to be re-evaluated, expending some power. 3571 */ 3572 static void ilk_write_wm_values(struct drm_i915_private *dev_priv, 3573 struct ilk_wm_values *results) 3574 { 3575 struct ilk_wm_values *previous = &dev_priv->wm.hw; 3576 unsigned int dirty; 3577 u32 val; 3578 3579 dirty = ilk_compute_wm_dirty(dev_priv, previous, results); 3580 if (!dirty) 3581 return; 3582 3583 _ilk_disable_lp_wm(dev_priv, dirty); 3584 3585 if (dirty & WM_DIRTY_PIPE(PIPE_A)) 3586 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]); 3587 if (dirty & WM_DIRTY_PIPE(PIPE_B)) 3588 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]); 3589 if (dirty & WM_DIRTY_PIPE(PIPE_C)) 3590 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]); 3591 3592 if (dirty & WM_DIRTY_LINETIME(PIPE_A)) 3593 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]); 3594 if (dirty & WM_DIRTY_LINETIME(PIPE_B)) 3595 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]); 3596 if (dirty & WM_DIRTY_LINETIME(PIPE_C)) 3597 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]); 3598 3599 if (dirty & WM_DIRTY_DDB) { 3600 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { 3601 val = I915_READ(WM_MISC); 3602 if (results->partitioning == INTEL_DDB_PART_1_2) 3603 val &= ~WM_MISC_DATA_PARTITION_5_6; 3604 else 3605 val |= WM_MISC_DATA_PARTITION_5_6; 3606 I915_WRITE(WM_MISC, val); 3607 } else { 3608 val = I915_READ(DISP_ARB_CTL2); 3609 if (results->partitioning == INTEL_DDB_PART_1_2) 3610 val &= ~DISP_DATA_PARTITION_5_6; 3611 else 3612 val |= DISP_DATA_PARTITION_5_6; 3613 I915_WRITE(DISP_ARB_CTL2, val); 3614 } 3615 } 3616 3617 if (dirty & WM_DIRTY_FBC) { 3618 val = I915_READ(DISP_ARB_CTL); 3619 if (results->enable_fbc_wm) 3620 val &= ~DISP_FBC_WM_DIS; 3621 else 3622 val |= DISP_FBC_WM_DIS; 3623 I915_WRITE(DISP_ARB_CTL, val); 3624 } 3625 3626 if (dirty & WM_DIRTY_LP(1) && 3627 previous->wm_lp_spr[0] != results->wm_lp_spr[0]) 3628 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]); 3629 3630 if (INTEL_GEN(dev_priv) >= 7) { 3631 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1]) 3632 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]); 3633 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2]) 3634 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]); 3635 } 3636 3637 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0]) 3638 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]); 3639 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1]) 3640 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]); 3641 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2]) 3642 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]); 3643 3644 dev_priv->wm.hw = *results; 3645 } 3646 3647 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv) 3648 { 3649 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL); 3650 } 3651 3652 static u8 intel_enabled_dbuf_slices_num(struct drm_i915_private *dev_priv) 3653 { 3654 u8 enabled_slices; 3655 3656 /* Slice 1 will always be enabled */ 3657 enabled_slices = 1; 3658 3659 /* Gen prior to GEN11 have only one DBuf slice */ 3660 if (INTEL_GEN(dev_priv) < 11) 3661 return enabled_slices; 3662 3663 /* 3664 * FIXME: for now we'll only ever use 1 slice; pretend that we have 3665 * only that 1 slice enabled until we have a proper way for on-demand 3666 * toggling of the second slice. 3667 */ 3668 if (0 && I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE) 3669 enabled_slices++; 3670 3671 return enabled_slices; 3672 } 3673 3674 /* 3675 * FIXME: We still don't have the proper code detect if we need to apply the WA, 3676 * so assume we'll always need it in order to avoid underruns. 3677 */ 3678 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv) 3679 { 3680 return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv); 3681 } 3682 3683 static bool 3684 intel_has_sagv(struct drm_i915_private *dev_priv) 3685 { 3686 /* HACK! */ 3687 if (IS_GEN(dev_priv, 12)) 3688 return false; 3689 3690 return (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) && 3691 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED; 3692 } 3693 3694 static void 3695 skl_setup_sagv_block_time(struct drm_i915_private *dev_priv) 3696 { 3697 if (INTEL_GEN(dev_priv) >= 12) { 3698 u32 val = 0; 3699 int ret; 3700 3701 ret = sandybridge_pcode_read(dev_priv, 3702 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US, 3703 &val, NULL); 3704 if (!ret) { 3705 dev_priv->sagv_block_time_us = val; 3706 return; 3707 } 3708 3709 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n"); 3710 } else if (IS_GEN(dev_priv, 11)) { 3711 dev_priv->sagv_block_time_us = 10; 3712 return; 3713 } else if (IS_GEN(dev_priv, 10)) { 3714 dev_priv->sagv_block_time_us = 20; 3715 return; 3716 } else if (IS_GEN(dev_priv, 9)) { 3717 dev_priv->sagv_block_time_us = 30; 3718 return; 3719 } else { 3720 MISSING_CASE(INTEL_GEN(dev_priv)); 3721 } 3722 3723 /* Default to an unusable block time */ 3724 dev_priv->sagv_block_time_us = -1; 3725 } 3726 3727 /* 3728 * SAGV dynamically adjusts the system agent voltage and clock frequencies 3729 * depending on power and performance requirements. The display engine access 3730 * to system memory is blocked during the adjustment time. Because of the 3731 * blocking time, having this enabled can cause full system hangs and/or pipe 3732 * underruns if we don't meet all of the following requirements: 3733 * 3734 * - <= 1 pipe enabled 3735 * - All planes can enable watermarks for latencies >= SAGV engine block time 3736 * - We're not using an interlaced display configuration 3737 */ 3738 int 3739 intel_enable_sagv(struct drm_i915_private *dev_priv) 3740 { 3741 int ret; 3742 3743 if (!intel_has_sagv(dev_priv)) 3744 return 0; 3745 3746 if (dev_priv->sagv_status == I915_SAGV_ENABLED) 3747 return 0; 3748 3749 drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n"); 3750 ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL, 3751 GEN9_SAGV_ENABLE); 3752 3753 /* We don't need to wait for SAGV when enabling */ 3754 3755 /* 3756 * Some skl systems, pre-release machines in particular, 3757 * don't actually have SAGV. 3758 */ 3759 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) { 3760 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n"); 3761 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED; 3762 return 0; 3763 } else if (ret < 0) { 3764 drm_err(&dev_priv->drm, "Failed to enable SAGV\n"); 3765 return ret; 3766 } 3767 3768 dev_priv->sagv_status = I915_SAGV_ENABLED; 3769 return 0; 3770 } 3771 3772 int 3773 intel_disable_sagv(struct drm_i915_private *dev_priv) 3774 { 3775 int ret; 3776 3777 if (!intel_has_sagv(dev_priv)) 3778 return 0; 3779 3780 if (dev_priv->sagv_status == I915_SAGV_DISABLED) 3781 return 0; 3782 3783 drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n"); 3784 /* bspec says to keep retrying for at least 1 ms */ 3785 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL, 3786 GEN9_SAGV_DISABLE, 3787 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED, 3788 1); 3789 /* 3790 * Some skl systems, pre-release machines in particular, 3791 * don't actually have SAGV. 3792 */ 3793 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) { 3794 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n"); 3795 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED; 3796 return 0; 3797 } else if (ret < 0) { 3798 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret); 3799 return ret; 3800 } 3801 3802 dev_priv->sagv_status = I915_SAGV_DISABLED; 3803 return 0; 3804 } 3805 3806 bool intel_can_enable_sagv(struct intel_atomic_state *state) 3807 { 3808 struct drm_device *dev = state->base.dev; 3809 struct drm_i915_private *dev_priv = to_i915(dev); 3810 struct intel_crtc *crtc; 3811 struct intel_plane *plane; 3812 struct intel_crtc_state *crtc_state; 3813 enum pipe pipe; 3814 int level, latency; 3815 3816 if (!intel_has_sagv(dev_priv)) 3817 return false; 3818 3819 /* 3820 * If there are no active CRTCs, no additional checks need be performed 3821 */ 3822 if (hweight8(state->active_pipes) == 0) 3823 return true; 3824 3825 /* 3826 * SKL+ workaround: bspec recommends we disable SAGV when we have 3827 * more then one pipe enabled 3828 */ 3829 if (hweight8(state->active_pipes) > 1) 3830 return false; 3831 3832 /* Since we're now guaranteed to only have one active CRTC... */ 3833 pipe = ffs(state->active_pipes) - 1; 3834 crtc = intel_get_crtc_for_pipe(dev_priv, pipe); 3835 crtc_state = to_intel_crtc_state(crtc->base.state); 3836 3837 if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) 3838 return false; 3839 3840 for_each_intel_plane_on_crtc(dev, crtc, plane) { 3841 struct skl_plane_wm *wm = 3842 &crtc_state->wm.skl.optimal.planes[plane->id]; 3843 3844 /* Skip this plane if it's not enabled */ 3845 if (!wm->wm[0].plane_en) 3846 continue; 3847 3848 /* Find the highest enabled wm level for this plane */ 3849 for (level = ilk_wm_max_level(dev_priv); 3850 !wm->wm[level].plane_en; --level) 3851 { } 3852 3853 latency = dev_priv->wm.skl_latency[level]; 3854 3855 if (skl_needs_memory_bw_wa(dev_priv) && 3856 plane->base.state->fb->modifier == 3857 I915_FORMAT_MOD_X_TILED) 3858 latency += 15; 3859 3860 /* 3861 * If any of the planes on this pipe don't enable wm levels that 3862 * incur memory latencies higher than sagv_block_time_us we 3863 * can't enable SAGV. 3864 */ 3865 if (latency < dev_priv->sagv_block_time_us) 3866 return false; 3867 } 3868 3869 return true; 3870 } 3871 3872 static u16 intel_get_ddb_size(struct drm_i915_private *dev_priv, 3873 const struct intel_crtc_state *crtc_state, 3874 const u64 total_data_rate, 3875 const int num_active, 3876 struct skl_ddb_allocation *ddb) 3877 { 3878 const struct drm_display_mode *adjusted_mode; 3879 u64 total_data_bw; 3880 u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size; 3881 3882 WARN_ON(ddb_size == 0); 3883 3884 if (INTEL_GEN(dev_priv) < 11) 3885 return ddb_size - 4; /* 4 blocks for bypass path allocation */ 3886 3887 adjusted_mode = &crtc_state->hw.adjusted_mode; 3888 total_data_bw = total_data_rate * drm_mode_vrefresh(adjusted_mode); 3889 3890 /* 3891 * 12GB/s is maximum BW supported by single DBuf slice. 3892 * 3893 * FIXME dbuf slice code is broken: 3894 * - must wait for planes to stop using the slice before powering it off 3895 * - plane straddling both slices is illegal in multi-pipe scenarios 3896 * - should validate we stay within the hw bandwidth limits 3897 */ 3898 if (0 && (num_active > 1 || total_data_bw >= GBps(12))) { 3899 ddb->enabled_slices = 2; 3900 } else { 3901 ddb->enabled_slices = 1; 3902 ddb_size /= 2; 3903 } 3904 3905 return ddb_size; 3906 } 3907 3908 static void 3909 skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv, 3910 const struct intel_crtc_state *crtc_state, 3911 const u64 total_data_rate, 3912 struct skl_ddb_allocation *ddb, 3913 struct skl_ddb_entry *alloc, /* out */ 3914 int *num_active /* out */) 3915 { 3916 struct drm_atomic_state *state = crtc_state->uapi.state; 3917 struct intel_atomic_state *intel_state = to_intel_atomic_state(state); 3918 struct drm_crtc *for_crtc = crtc_state->uapi.crtc; 3919 const struct intel_crtc *crtc; 3920 u32 pipe_width = 0, total_width = 0, width_before_pipe = 0; 3921 enum pipe for_pipe = to_intel_crtc(for_crtc)->pipe; 3922 u16 ddb_size; 3923 u32 i; 3924 3925 if (WARN_ON(!state) || !crtc_state->hw.active) { 3926 alloc->start = 0; 3927 alloc->end = 0; 3928 *num_active = hweight8(dev_priv->active_pipes); 3929 return; 3930 } 3931 3932 if (intel_state->active_pipe_changes) 3933 *num_active = hweight8(intel_state->active_pipes); 3934 else 3935 *num_active = hweight8(dev_priv->active_pipes); 3936 3937 ddb_size = intel_get_ddb_size(dev_priv, crtc_state, total_data_rate, 3938 *num_active, ddb); 3939 3940 /* 3941 * If the state doesn't change the active CRTC's or there is no 3942 * modeset request, then there's no need to recalculate; 3943 * the existing pipe allocation limits should remain unchanged. 3944 * Note that we're safe from racing commits since any racing commit 3945 * that changes the active CRTC list or do modeset would need to 3946 * grab _all_ crtc locks, including the one we currently hold. 3947 */ 3948 if (!intel_state->active_pipe_changes && !intel_state->modeset) { 3949 /* 3950 * alloc may be cleared by clear_intel_crtc_state, 3951 * copy from old state to be sure 3952 */ 3953 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb; 3954 return; 3955 } 3956 3957 /* 3958 * Watermark/ddb requirement highly depends upon width of the 3959 * framebuffer, So instead of allocating DDB equally among pipes 3960 * distribute DDB based on resolution/width of the display. 3961 */ 3962 for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) { 3963 const struct drm_display_mode *adjusted_mode = 3964 &crtc_state->hw.adjusted_mode; 3965 enum pipe pipe = crtc->pipe; 3966 int hdisplay, vdisplay; 3967 3968 if (!crtc_state->hw.enable) 3969 continue; 3970 3971 drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay); 3972 total_width += hdisplay; 3973 3974 if (pipe < for_pipe) 3975 width_before_pipe += hdisplay; 3976 else if (pipe == for_pipe) 3977 pipe_width = hdisplay; 3978 } 3979 3980 alloc->start = ddb_size * width_before_pipe / total_width; 3981 alloc->end = ddb_size * (width_before_pipe + pipe_width) / total_width; 3982 } 3983 3984 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state, 3985 int width, const struct drm_format_info *format, 3986 u64 modifier, unsigned int rotation, 3987 u32 plane_pixel_rate, struct skl_wm_params *wp, 3988 int color_plane); 3989 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, 3990 int level, 3991 const struct skl_wm_params *wp, 3992 const struct skl_wm_level *result_prev, 3993 struct skl_wm_level *result /* out */); 3994 3995 static unsigned int 3996 skl_cursor_allocation(const struct intel_crtc_state *crtc_state, 3997 int num_active) 3998 { 3999 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 4000 int level, max_level = ilk_wm_max_level(dev_priv); 4001 struct skl_wm_level wm = {}; 4002 int ret, min_ddb_alloc = 0; 4003 struct skl_wm_params wp; 4004 4005 ret = skl_compute_wm_params(crtc_state, 256, 4006 drm_format_info(DRM_FORMAT_ARGB8888), 4007 DRM_FORMAT_MOD_LINEAR, 4008 DRM_MODE_ROTATE_0, 4009 crtc_state->pixel_rate, &wp, 0); 4010 WARN_ON(ret); 4011 4012 for (level = 0; level <= max_level; level++) { 4013 skl_compute_plane_wm(crtc_state, level, &wp, &wm, &wm); 4014 if (wm.min_ddb_alloc == U16_MAX) 4015 break; 4016 4017 min_ddb_alloc = wm.min_ddb_alloc; 4018 } 4019 4020 return max(num_active == 1 ? 32 : 8, min_ddb_alloc); 4021 } 4022 4023 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv, 4024 struct skl_ddb_entry *entry, u32 reg) 4025 { 4026 4027 entry->start = reg & DDB_ENTRY_MASK; 4028 entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK; 4029 4030 if (entry->end) 4031 entry->end += 1; 4032 } 4033 4034 static void 4035 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv, 4036 const enum pipe pipe, 4037 const enum plane_id plane_id, 4038 struct skl_ddb_entry *ddb_y, 4039 struct skl_ddb_entry *ddb_uv) 4040 { 4041 u32 val, val2; 4042 u32 fourcc = 0; 4043 4044 /* Cursor doesn't support NV12/planar, so no extra calculation needed */ 4045 if (plane_id == PLANE_CURSOR) { 4046 val = I915_READ(CUR_BUF_CFG(pipe)); 4047 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val); 4048 return; 4049 } 4050 4051 val = I915_READ(PLANE_CTL(pipe, plane_id)); 4052 4053 /* No DDB allocated for disabled planes */ 4054 if (val & PLANE_CTL_ENABLE) 4055 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK, 4056 val & PLANE_CTL_ORDER_RGBX, 4057 val & PLANE_CTL_ALPHA_MASK); 4058 4059 if (INTEL_GEN(dev_priv) >= 11) { 4060 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id)); 4061 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val); 4062 } else { 4063 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id)); 4064 val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id)); 4065 4066 if (fourcc && 4067 drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc))) 4068 swap(val, val2); 4069 4070 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val); 4071 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2); 4072 } 4073 } 4074 4075 void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc, 4076 struct skl_ddb_entry *ddb_y, 4077 struct skl_ddb_entry *ddb_uv) 4078 { 4079 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 4080 enum intel_display_power_domain power_domain; 4081 enum pipe pipe = crtc->pipe; 4082 intel_wakeref_t wakeref; 4083 enum plane_id plane_id; 4084 4085 power_domain = POWER_DOMAIN_PIPE(pipe); 4086 wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); 4087 if (!wakeref) 4088 return; 4089 4090 for_each_plane_id_on_crtc(crtc, plane_id) 4091 skl_ddb_get_hw_plane_state(dev_priv, pipe, 4092 plane_id, 4093 &ddb_y[plane_id], 4094 &ddb_uv[plane_id]); 4095 4096 intel_display_power_put(dev_priv, power_domain, wakeref); 4097 } 4098 4099 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv, 4100 struct skl_ddb_allocation *ddb /* out */) 4101 { 4102 ddb->enabled_slices = intel_enabled_dbuf_slices_num(dev_priv); 4103 } 4104 4105 /* 4106 * Determines the downscale amount of a plane for the purposes of watermark calculations. 4107 * The bspec defines downscale amount as: 4108 * 4109 * """ 4110 * Horizontal down scale amount = maximum[1, Horizontal source size / 4111 * Horizontal destination size] 4112 * Vertical down scale amount = maximum[1, Vertical source size / 4113 * Vertical destination size] 4114 * Total down scale amount = Horizontal down scale amount * 4115 * Vertical down scale amount 4116 * """ 4117 * 4118 * Return value is provided in 16.16 fixed point form to retain fractional part. 4119 * Caller should take care of dividing & rounding off the value. 4120 */ 4121 static uint_fixed_16_16_t 4122 skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state, 4123 const struct intel_plane_state *plane_state) 4124 { 4125 u32 src_w, src_h, dst_w, dst_h; 4126 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio; 4127 uint_fixed_16_16_t downscale_h, downscale_w; 4128 4129 if (WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state))) 4130 return u32_to_fixed16(0); 4131 4132 /* 4133 * Src coordinates are already rotated by 270 degrees for 4134 * the 90/270 degree plane rotation cases (to match the 4135 * GTT mapping), hence no need to account for rotation here. 4136 * 4137 * n.b., src is 16.16 fixed point, dst is whole integer. 4138 */ 4139 src_w = drm_rect_width(&plane_state->uapi.src) >> 16; 4140 src_h = drm_rect_height(&plane_state->uapi.src) >> 16; 4141 dst_w = drm_rect_width(&plane_state->uapi.dst); 4142 dst_h = drm_rect_height(&plane_state->uapi.dst); 4143 4144 fp_w_ratio = div_fixed16(src_w, dst_w); 4145 fp_h_ratio = div_fixed16(src_h, dst_h); 4146 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1)); 4147 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1)); 4148 4149 return mul_fixed16(downscale_w, downscale_h); 4150 } 4151 4152 static u64 4153 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state, 4154 const struct intel_plane_state *plane_state, 4155 int color_plane) 4156 { 4157 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 4158 const struct drm_framebuffer *fb = plane_state->hw.fb; 4159 u32 data_rate; 4160 u32 width = 0, height = 0; 4161 uint_fixed_16_16_t down_scale_amount; 4162 u64 rate; 4163 4164 if (!plane_state->uapi.visible) 4165 return 0; 4166 4167 if (plane->id == PLANE_CURSOR) 4168 return 0; 4169 4170 if (color_plane == 1 && 4171 !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier)) 4172 return 0; 4173 4174 /* 4175 * Src coordinates are already rotated by 270 degrees for 4176 * the 90/270 degree plane rotation cases (to match the 4177 * GTT mapping), hence no need to account for rotation here. 4178 */ 4179 width = drm_rect_width(&plane_state->uapi.src) >> 16; 4180 height = drm_rect_height(&plane_state->uapi.src) >> 16; 4181 4182 /* UV plane does 1/2 pixel sub-sampling */ 4183 if (color_plane == 1) { 4184 width /= 2; 4185 height /= 2; 4186 } 4187 4188 data_rate = width * height; 4189 4190 down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state); 4191 4192 rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount); 4193 4194 rate *= fb->format->cpp[color_plane]; 4195 return rate; 4196 } 4197 4198 static u64 4199 skl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state, 4200 u64 *plane_data_rate, 4201 u64 *uv_plane_data_rate) 4202 { 4203 struct drm_atomic_state *state = crtc_state->uapi.state; 4204 struct intel_plane *plane; 4205 const struct intel_plane_state *plane_state; 4206 u64 total_data_rate = 0; 4207 4208 if (WARN_ON(!state)) 4209 return 0; 4210 4211 /* Calculate and cache data rate for each plane */ 4212 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) { 4213 enum plane_id plane_id = plane->id; 4214 u64 rate; 4215 4216 /* packed/y */ 4217 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0); 4218 plane_data_rate[plane_id] = rate; 4219 total_data_rate += rate; 4220 4221 /* uv-plane */ 4222 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1); 4223 uv_plane_data_rate[plane_id] = rate; 4224 total_data_rate += rate; 4225 } 4226 4227 return total_data_rate; 4228 } 4229 4230 static u64 4231 icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state, 4232 u64 *plane_data_rate) 4233 { 4234 struct intel_plane *plane; 4235 const struct intel_plane_state *plane_state; 4236 u64 total_data_rate = 0; 4237 4238 if (WARN_ON(!crtc_state->uapi.state)) 4239 return 0; 4240 4241 /* Calculate and cache data rate for each plane */ 4242 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) { 4243 enum plane_id plane_id = plane->id; 4244 u64 rate; 4245 4246 if (!plane_state->planar_linked_plane) { 4247 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0); 4248 plane_data_rate[plane_id] = rate; 4249 total_data_rate += rate; 4250 } else { 4251 enum plane_id y_plane_id; 4252 4253 /* 4254 * The slave plane might not iterate in 4255 * intel_atomic_crtc_state_for_each_plane_state(), 4256 * and needs the master plane state which may be 4257 * NULL if we try get_new_plane_state(), so we 4258 * always calculate from the master. 4259 */ 4260 if (plane_state->planar_slave) 4261 continue; 4262 4263 /* Y plane rate is calculated on the slave */ 4264 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0); 4265 y_plane_id = plane_state->planar_linked_plane->id; 4266 plane_data_rate[y_plane_id] = rate; 4267 total_data_rate += rate; 4268 4269 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1); 4270 plane_data_rate[plane_id] = rate; 4271 total_data_rate += rate; 4272 } 4273 } 4274 4275 return total_data_rate; 4276 } 4277 4278 static int 4279 skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state, 4280 struct skl_ddb_allocation *ddb /* out */) 4281 { 4282 struct drm_atomic_state *state = crtc_state->uapi.state; 4283 struct drm_crtc *crtc = crtc_state->uapi.crtc; 4284 struct drm_i915_private *dev_priv = to_i915(crtc->dev); 4285 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 4286 struct skl_ddb_entry *alloc = &crtc_state->wm.skl.ddb; 4287 u16 alloc_size, start = 0; 4288 u16 total[I915_MAX_PLANES] = {}; 4289 u16 uv_total[I915_MAX_PLANES] = {}; 4290 u64 total_data_rate; 4291 enum plane_id plane_id; 4292 int num_active; 4293 u64 plane_data_rate[I915_MAX_PLANES] = {}; 4294 u64 uv_plane_data_rate[I915_MAX_PLANES] = {}; 4295 u32 blocks; 4296 int level; 4297 4298 /* Clear the partitioning for disabled planes. */ 4299 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y)); 4300 memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv)); 4301 4302 if (WARN_ON(!state)) 4303 return 0; 4304 4305 if (!crtc_state->hw.active) { 4306 alloc->start = alloc->end = 0; 4307 return 0; 4308 } 4309 4310 if (INTEL_GEN(dev_priv) >= 11) 4311 total_data_rate = 4312 icl_get_total_relative_data_rate(crtc_state, 4313 plane_data_rate); 4314 else 4315 total_data_rate = 4316 skl_get_total_relative_data_rate(crtc_state, 4317 plane_data_rate, 4318 uv_plane_data_rate); 4319 4320 4321 skl_ddb_get_pipe_allocation_limits(dev_priv, crtc_state, total_data_rate, 4322 ddb, alloc, &num_active); 4323 alloc_size = skl_ddb_entry_size(alloc); 4324 if (alloc_size == 0) 4325 return 0; 4326 4327 /* Allocate fixed number of blocks for cursor. */ 4328 total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active); 4329 alloc_size -= total[PLANE_CURSOR]; 4330 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start = 4331 alloc->end - total[PLANE_CURSOR]; 4332 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end; 4333 4334 if (total_data_rate == 0) 4335 return 0; 4336 4337 /* 4338 * Find the highest watermark level for which we can satisfy the block 4339 * requirement of active planes. 4340 */ 4341 for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) { 4342 blocks = 0; 4343 for_each_plane_id_on_crtc(intel_crtc, plane_id) { 4344 const struct skl_plane_wm *wm = 4345 &crtc_state->wm.skl.optimal.planes[plane_id]; 4346 4347 if (plane_id == PLANE_CURSOR) { 4348 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) { 4349 WARN_ON(wm->wm[level].min_ddb_alloc != U16_MAX); 4350 blocks = U32_MAX; 4351 break; 4352 } 4353 continue; 4354 } 4355 4356 blocks += wm->wm[level].min_ddb_alloc; 4357 blocks += wm->uv_wm[level].min_ddb_alloc; 4358 } 4359 4360 if (blocks <= alloc_size) { 4361 alloc_size -= blocks; 4362 break; 4363 } 4364 } 4365 4366 if (level < 0) { 4367 drm_dbg_kms(&dev_priv->drm, 4368 "Requested display configuration exceeds system DDB limitations"); 4369 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n", 4370 blocks, alloc_size); 4371 return -EINVAL; 4372 } 4373 4374 /* 4375 * Grant each plane the blocks it requires at the highest achievable 4376 * watermark level, plus an extra share of the leftover blocks 4377 * proportional to its relative data rate. 4378 */ 4379 for_each_plane_id_on_crtc(intel_crtc, plane_id) { 4380 const struct skl_plane_wm *wm = 4381 &crtc_state->wm.skl.optimal.planes[plane_id]; 4382 u64 rate; 4383 u16 extra; 4384 4385 if (plane_id == PLANE_CURSOR) 4386 continue; 4387 4388 /* 4389 * We've accounted for all active planes; remaining planes are 4390 * all disabled. 4391 */ 4392 if (total_data_rate == 0) 4393 break; 4394 4395 rate = plane_data_rate[plane_id]; 4396 extra = min_t(u16, alloc_size, 4397 DIV64_U64_ROUND_UP(alloc_size * rate, 4398 total_data_rate)); 4399 total[plane_id] = wm->wm[level].min_ddb_alloc + extra; 4400 alloc_size -= extra; 4401 total_data_rate -= rate; 4402 4403 if (total_data_rate == 0) 4404 break; 4405 4406 rate = uv_plane_data_rate[plane_id]; 4407 extra = min_t(u16, alloc_size, 4408 DIV64_U64_ROUND_UP(alloc_size * rate, 4409 total_data_rate)); 4410 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra; 4411 alloc_size -= extra; 4412 total_data_rate -= rate; 4413 } 4414 WARN_ON(alloc_size != 0 || total_data_rate != 0); 4415 4416 /* Set the actual DDB start/end points for each plane */ 4417 start = alloc->start; 4418 for_each_plane_id_on_crtc(intel_crtc, plane_id) { 4419 struct skl_ddb_entry *plane_alloc = 4420 &crtc_state->wm.skl.plane_ddb_y[plane_id]; 4421 struct skl_ddb_entry *uv_plane_alloc = 4422 &crtc_state->wm.skl.plane_ddb_uv[plane_id]; 4423 4424 if (plane_id == PLANE_CURSOR) 4425 continue; 4426 4427 /* Gen11+ uses a separate plane for UV watermarks */ 4428 WARN_ON(INTEL_GEN(dev_priv) >= 11 && uv_total[plane_id]); 4429 4430 /* Leave disabled planes at (0,0) */ 4431 if (total[plane_id]) { 4432 plane_alloc->start = start; 4433 start += total[plane_id]; 4434 plane_alloc->end = start; 4435 } 4436 4437 if (uv_total[plane_id]) { 4438 uv_plane_alloc->start = start; 4439 start += uv_total[plane_id]; 4440 uv_plane_alloc->end = start; 4441 } 4442 } 4443 4444 /* 4445 * When we calculated watermark values we didn't know how high 4446 * of a level we'd actually be able to hit, so we just marked 4447 * all levels as "enabled." Go back now and disable the ones 4448 * that aren't actually possible. 4449 */ 4450 for (level++; level <= ilk_wm_max_level(dev_priv); level++) { 4451 for_each_plane_id_on_crtc(intel_crtc, plane_id) { 4452 struct skl_plane_wm *wm = 4453 &crtc_state->wm.skl.optimal.planes[plane_id]; 4454 4455 /* 4456 * We only disable the watermarks for each plane if 4457 * they exceed the ddb allocation of said plane. This 4458 * is done so that we don't end up touching cursor 4459 * watermarks needlessly when some other plane reduces 4460 * our max possible watermark level. 4461 * 4462 * Bspec has this to say about the PLANE_WM enable bit: 4463 * "All the watermarks at this level for all enabled 4464 * planes must be enabled before the level will be used." 4465 * So this is actually safe to do. 4466 */ 4467 if (wm->wm[level].min_ddb_alloc > total[plane_id] || 4468 wm->uv_wm[level].min_ddb_alloc > uv_total[plane_id]) 4469 memset(&wm->wm[level], 0, sizeof(wm->wm[level])); 4470 4471 /* 4472 * Wa_1408961008:icl, ehl 4473 * Underruns with WM1+ disabled 4474 */ 4475 if (IS_GEN(dev_priv, 11) && 4476 level == 1 && wm->wm[0].plane_en) { 4477 wm->wm[level].plane_res_b = wm->wm[0].plane_res_b; 4478 wm->wm[level].plane_res_l = wm->wm[0].plane_res_l; 4479 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines; 4480 } 4481 } 4482 } 4483 4484 /* 4485 * Go back and disable the transition watermark if it turns out we 4486 * don't have enough DDB blocks for it. 4487 */ 4488 for_each_plane_id_on_crtc(intel_crtc, plane_id) { 4489 struct skl_plane_wm *wm = 4490 &crtc_state->wm.skl.optimal.planes[plane_id]; 4491 4492 if (wm->trans_wm.plane_res_b >= total[plane_id]) 4493 memset(&wm->trans_wm, 0, sizeof(wm->trans_wm)); 4494 } 4495 4496 return 0; 4497 } 4498 4499 /* 4500 * The max latency should be 257 (max the punit can code is 255 and we add 2us 4501 * for the read latency) and cpp should always be <= 8, so that 4502 * should allow pixel_rate up to ~2 GHz which seems sufficient since max 4503 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that. 4504 */ 4505 static uint_fixed_16_16_t 4506 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate, 4507 u8 cpp, u32 latency, u32 dbuf_block_size) 4508 { 4509 u32 wm_intermediate_val; 4510 uint_fixed_16_16_t ret; 4511 4512 if (latency == 0) 4513 return FP_16_16_MAX; 4514 4515 wm_intermediate_val = latency * pixel_rate * cpp; 4516 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size); 4517 4518 if (INTEL_GEN(dev_priv) >= 10) 4519 ret = add_fixed16_u32(ret, 1); 4520 4521 return ret; 4522 } 4523 4524 static uint_fixed_16_16_t 4525 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency, 4526 uint_fixed_16_16_t plane_blocks_per_line) 4527 { 4528 u32 wm_intermediate_val; 4529 uint_fixed_16_16_t ret; 4530 4531 if (latency == 0) 4532 return FP_16_16_MAX; 4533 4534 wm_intermediate_val = latency * pixel_rate; 4535 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val, 4536 pipe_htotal * 1000); 4537 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line); 4538 return ret; 4539 } 4540 4541 static uint_fixed_16_16_t 4542 intel_get_linetime_us(const struct intel_crtc_state *crtc_state) 4543 { 4544 u32 pixel_rate; 4545 u32 crtc_htotal; 4546 uint_fixed_16_16_t linetime_us; 4547 4548 if (!crtc_state->hw.active) 4549 return u32_to_fixed16(0); 4550 4551 pixel_rate = crtc_state->pixel_rate; 4552 4553 if (WARN_ON(pixel_rate == 0)) 4554 return u32_to_fixed16(0); 4555 4556 crtc_htotal = crtc_state->hw.adjusted_mode.crtc_htotal; 4557 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate); 4558 4559 return linetime_us; 4560 } 4561 4562 static u32 4563 skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *crtc_state, 4564 const struct intel_plane_state *plane_state) 4565 { 4566 u64 adjusted_pixel_rate; 4567 uint_fixed_16_16_t downscale_amount; 4568 4569 /* Shouldn't reach here on disabled planes... */ 4570 if (WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state))) 4571 return 0; 4572 4573 /* 4574 * Adjusted plane pixel rate is just the pipe's adjusted pixel rate 4575 * with additional adjustments for plane-specific scaling. 4576 */ 4577 adjusted_pixel_rate = crtc_state->pixel_rate; 4578 downscale_amount = skl_plane_downscale_amount(crtc_state, plane_state); 4579 4580 return mul_round_up_u32_fixed16(adjusted_pixel_rate, 4581 downscale_amount); 4582 } 4583 4584 static int 4585 skl_compute_wm_params(const struct intel_crtc_state *crtc_state, 4586 int width, const struct drm_format_info *format, 4587 u64 modifier, unsigned int rotation, 4588 u32 plane_pixel_rate, struct skl_wm_params *wp, 4589 int color_plane) 4590 { 4591 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 4592 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 4593 u32 interm_pbpl; 4594 4595 /* only planar format has two planes */ 4596 if (color_plane == 1 && 4597 !intel_format_info_is_yuv_semiplanar(format, modifier)) { 4598 drm_dbg_kms(&dev_priv->drm, 4599 "Non planar format have single plane\n"); 4600 return -EINVAL; 4601 } 4602 4603 wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED || 4604 modifier == I915_FORMAT_MOD_Yf_TILED || 4605 modifier == I915_FORMAT_MOD_Y_TILED_CCS || 4606 modifier == I915_FORMAT_MOD_Yf_TILED_CCS; 4607 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED; 4608 wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS || 4609 modifier == I915_FORMAT_MOD_Yf_TILED_CCS; 4610 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier); 4611 4612 wp->width = width; 4613 if (color_plane == 1 && wp->is_planar) 4614 wp->width /= 2; 4615 4616 wp->cpp = format->cpp[color_plane]; 4617 wp->plane_pixel_rate = plane_pixel_rate; 4618 4619 if (INTEL_GEN(dev_priv) >= 11 && 4620 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1) 4621 wp->dbuf_block_size = 256; 4622 else 4623 wp->dbuf_block_size = 512; 4624 4625 if (drm_rotation_90_or_270(rotation)) { 4626 switch (wp->cpp) { 4627 case 1: 4628 wp->y_min_scanlines = 16; 4629 break; 4630 case 2: 4631 wp->y_min_scanlines = 8; 4632 break; 4633 case 4: 4634 wp->y_min_scanlines = 4; 4635 break; 4636 default: 4637 MISSING_CASE(wp->cpp); 4638 return -EINVAL; 4639 } 4640 } else { 4641 wp->y_min_scanlines = 4; 4642 } 4643 4644 if (skl_needs_memory_bw_wa(dev_priv)) 4645 wp->y_min_scanlines *= 2; 4646 4647 wp->plane_bytes_per_line = wp->width * wp->cpp; 4648 if (wp->y_tiled) { 4649 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line * 4650 wp->y_min_scanlines, 4651 wp->dbuf_block_size); 4652 4653 if (INTEL_GEN(dev_priv) >= 10) 4654 interm_pbpl++; 4655 4656 wp->plane_blocks_per_line = div_fixed16(interm_pbpl, 4657 wp->y_min_scanlines); 4658 } else if (wp->x_tiled && IS_GEN(dev_priv, 9)) { 4659 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line, 4660 wp->dbuf_block_size); 4661 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl); 4662 } else { 4663 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line, 4664 wp->dbuf_block_size) + 1; 4665 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl); 4666 } 4667 4668 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines, 4669 wp->plane_blocks_per_line); 4670 4671 wp->linetime_us = fixed16_to_u32_round_up( 4672 intel_get_linetime_us(crtc_state)); 4673 4674 return 0; 4675 } 4676 4677 static int 4678 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state, 4679 const struct intel_plane_state *plane_state, 4680 struct skl_wm_params *wp, int color_plane) 4681 { 4682 const struct drm_framebuffer *fb = plane_state->hw.fb; 4683 int width; 4684 4685 /* 4686 * Src coordinates are already rotated by 270 degrees for 4687 * the 90/270 degree plane rotation cases (to match the 4688 * GTT mapping), hence no need to account for rotation here. 4689 */ 4690 width = drm_rect_width(&plane_state->uapi.src) >> 16; 4691 4692 return skl_compute_wm_params(crtc_state, width, 4693 fb->format, fb->modifier, 4694 plane_state->hw.rotation, 4695 skl_adjusted_plane_pixel_rate(crtc_state, plane_state), 4696 wp, color_plane); 4697 } 4698 4699 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level) 4700 { 4701 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) 4702 return true; 4703 4704 /* The number of lines are ignored for the level 0 watermark. */ 4705 return level > 0; 4706 } 4707 4708 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, 4709 int level, 4710 const struct skl_wm_params *wp, 4711 const struct skl_wm_level *result_prev, 4712 struct skl_wm_level *result /* out */) 4713 { 4714 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 4715 u32 latency = dev_priv->wm.skl_latency[level]; 4716 uint_fixed_16_16_t method1, method2; 4717 uint_fixed_16_16_t selected_result; 4718 u32 res_blocks, res_lines, min_ddb_alloc = 0; 4719 4720 if (latency == 0) { 4721 /* reject it */ 4722 result->min_ddb_alloc = U16_MAX; 4723 return; 4724 } 4725 4726 /* 4727 * WaIncreaseLatencyIPCEnabled: kbl,cfl 4728 * Display WA #1141: kbl,cfl 4729 */ 4730 if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) || 4731 dev_priv->ipc_enabled) 4732 latency += 4; 4733 4734 if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled) 4735 latency += 15; 4736 4737 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate, 4738 wp->cpp, latency, wp->dbuf_block_size); 4739 method2 = skl_wm_method2(wp->plane_pixel_rate, 4740 crtc_state->hw.adjusted_mode.crtc_htotal, 4741 latency, 4742 wp->plane_blocks_per_line); 4743 4744 if (wp->y_tiled) { 4745 selected_result = max_fixed16(method2, wp->y_tile_minimum); 4746 } else { 4747 if ((wp->cpp * crtc_state->hw.adjusted_mode.crtc_htotal / 4748 wp->dbuf_block_size < 1) && 4749 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) { 4750 selected_result = method2; 4751 } else if (latency >= wp->linetime_us) { 4752 if (IS_GEN(dev_priv, 9) && 4753 !IS_GEMINILAKE(dev_priv)) 4754 selected_result = min_fixed16(method1, method2); 4755 else 4756 selected_result = method2; 4757 } else { 4758 selected_result = method1; 4759 } 4760 } 4761 4762 res_blocks = fixed16_to_u32_round_up(selected_result) + 1; 4763 res_lines = div_round_up_fixed16(selected_result, 4764 wp->plane_blocks_per_line); 4765 4766 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) { 4767 /* Display WA #1125: skl,bxt,kbl */ 4768 if (level == 0 && wp->rc_surface) 4769 res_blocks += 4770 fixed16_to_u32_round_up(wp->y_tile_minimum); 4771 4772 /* Display WA #1126: skl,bxt,kbl */ 4773 if (level >= 1 && level <= 7) { 4774 if (wp->y_tiled) { 4775 res_blocks += 4776 fixed16_to_u32_round_up(wp->y_tile_minimum); 4777 res_lines += wp->y_min_scanlines; 4778 } else { 4779 res_blocks++; 4780 } 4781 4782 /* 4783 * Make sure result blocks for higher latency levels are 4784 * atleast as high as level below the current level. 4785 * Assumption in DDB algorithm optimization for special 4786 * cases. Also covers Display WA #1125 for RC. 4787 */ 4788 if (result_prev->plane_res_b > res_blocks) 4789 res_blocks = result_prev->plane_res_b; 4790 } 4791 } 4792 4793 if (INTEL_GEN(dev_priv) >= 11) { 4794 if (wp->y_tiled) { 4795 int extra_lines; 4796 4797 if (res_lines % wp->y_min_scanlines == 0) 4798 extra_lines = wp->y_min_scanlines; 4799 else 4800 extra_lines = wp->y_min_scanlines * 2 - 4801 res_lines % wp->y_min_scanlines; 4802 4803 min_ddb_alloc = mul_round_up_u32_fixed16(res_lines + extra_lines, 4804 wp->plane_blocks_per_line); 4805 } else { 4806 min_ddb_alloc = res_blocks + 4807 DIV_ROUND_UP(res_blocks, 10); 4808 } 4809 } 4810 4811 if (!skl_wm_has_lines(dev_priv, level)) 4812 res_lines = 0; 4813 4814 if (res_lines > 31) { 4815 /* reject it */ 4816 result->min_ddb_alloc = U16_MAX; 4817 return; 4818 } 4819 4820 /* 4821 * If res_lines is valid, assume we can use this watermark level 4822 * for now. We'll come back and disable it after we calculate the 4823 * DDB allocation if it turns out we don't actually have enough 4824 * blocks to satisfy it. 4825 */ 4826 result->plane_res_b = res_blocks; 4827 result->plane_res_l = res_lines; 4828 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */ 4829 result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1; 4830 result->plane_en = true; 4831 } 4832 4833 static void 4834 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state, 4835 const struct skl_wm_params *wm_params, 4836 struct skl_wm_level *levels) 4837 { 4838 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 4839 int level, max_level = ilk_wm_max_level(dev_priv); 4840 struct skl_wm_level *result_prev = &levels[0]; 4841 4842 for (level = 0; level <= max_level; level++) { 4843 struct skl_wm_level *result = &levels[level]; 4844 4845 skl_compute_plane_wm(crtc_state, level, wm_params, 4846 result_prev, result); 4847 4848 result_prev = result; 4849 } 4850 } 4851 4852 static u32 4853 skl_compute_linetime_wm(const struct intel_crtc_state *crtc_state) 4854 { 4855 struct drm_atomic_state *state = crtc_state->uapi.state; 4856 struct drm_i915_private *dev_priv = to_i915(state->dev); 4857 uint_fixed_16_16_t linetime_us; 4858 u32 linetime_wm; 4859 4860 linetime_us = intel_get_linetime_us(crtc_state); 4861 linetime_wm = fixed16_to_u32_round_up(mul_u32_fixed16(8, linetime_us)); 4862 4863 /* Display WA #1135: BXT:ALL GLK:ALL */ 4864 if (IS_GEN9_LP(dev_priv) && dev_priv->ipc_enabled) 4865 linetime_wm /= 2; 4866 4867 return linetime_wm; 4868 } 4869 4870 static void skl_compute_transition_wm(const struct intel_crtc_state *crtc_state, 4871 const struct skl_wm_params *wp, 4872 struct skl_plane_wm *wm) 4873 { 4874 struct drm_device *dev = crtc_state->uapi.crtc->dev; 4875 const struct drm_i915_private *dev_priv = to_i915(dev); 4876 u16 trans_min, trans_y_tile_min; 4877 const u16 trans_amount = 10; /* This is configurable amount */ 4878 u16 wm0_sel_res_b, trans_offset_b, res_blocks; 4879 4880 /* Transition WM are not recommended by HW team for GEN9 */ 4881 if (INTEL_GEN(dev_priv) <= 9) 4882 return; 4883 4884 /* Transition WM don't make any sense if ipc is disabled */ 4885 if (!dev_priv->ipc_enabled) 4886 return; 4887 4888 trans_min = 14; 4889 if (INTEL_GEN(dev_priv) >= 11) 4890 trans_min = 4; 4891 4892 trans_offset_b = trans_min + trans_amount; 4893 4894 /* 4895 * The spec asks for Selected Result Blocks for wm0 (the real value), 4896 * not Result Blocks (the integer value). Pay attention to the capital 4897 * letters. The value wm_l0->plane_res_b is actually Result Blocks, but 4898 * since Result Blocks is the ceiling of Selected Result Blocks plus 1, 4899 * and since we later will have to get the ceiling of the sum in the 4900 * transition watermarks calculation, we can just pretend Selected 4901 * Result Blocks is Result Blocks minus 1 and it should work for the 4902 * current platforms. 4903 */ 4904 wm0_sel_res_b = wm->wm[0].plane_res_b - 1; 4905 4906 if (wp->y_tiled) { 4907 trans_y_tile_min = 4908 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum); 4909 res_blocks = max(wm0_sel_res_b, trans_y_tile_min) + 4910 trans_offset_b; 4911 } else { 4912 res_blocks = wm0_sel_res_b + trans_offset_b; 4913 4914 /* WA BUG:1938466 add one block for non y-tile planes */ 4915 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0)) 4916 res_blocks += 1; 4917 4918 } 4919 4920 /* 4921 * Just assume we can enable the transition watermark. After 4922 * computing the DDB we'll come back and disable it if that 4923 * assumption turns out to be false. 4924 */ 4925 wm->trans_wm.plane_res_b = res_blocks + 1; 4926 wm->trans_wm.plane_en = true; 4927 } 4928 4929 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state, 4930 const struct intel_plane_state *plane_state, 4931 enum plane_id plane_id, int color_plane) 4932 { 4933 struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id]; 4934 struct skl_wm_params wm_params; 4935 int ret; 4936 4937 ret = skl_compute_plane_wm_params(crtc_state, plane_state, 4938 &wm_params, color_plane); 4939 if (ret) 4940 return ret; 4941 4942 skl_compute_wm_levels(crtc_state, &wm_params, wm->wm); 4943 skl_compute_transition_wm(crtc_state, &wm_params, wm); 4944 4945 return 0; 4946 } 4947 4948 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state, 4949 const struct intel_plane_state *plane_state, 4950 enum plane_id plane_id) 4951 { 4952 struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id]; 4953 struct skl_wm_params wm_params; 4954 int ret; 4955 4956 wm->is_planar = true; 4957 4958 /* uv plane watermarks must also be validated for NV12/Planar */ 4959 ret = skl_compute_plane_wm_params(crtc_state, plane_state, 4960 &wm_params, 1); 4961 if (ret) 4962 return ret; 4963 4964 skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm); 4965 4966 return 0; 4967 } 4968 4969 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state, 4970 const struct intel_plane_state *plane_state) 4971 { 4972 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 4973 const struct drm_framebuffer *fb = plane_state->hw.fb; 4974 enum plane_id plane_id = plane->id; 4975 int ret; 4976 4977 if (!intel_wm_plane_visible(crtc_state, plane_state)) 4978 return 0; 4979 4980 ret = skl_build_plane_wm_single(crtc_state, plane_state, 4981 plane_id, 0); 4982 if (ret) 4983 return ret; 4984 4985 if (fb->format->is_yuv && fb->format->num_planes > 1) { 4986 ret = skl_build_plane_wm_uv(crtc_state, plane_state, 4987 plane_id); 4988 if (ret) 4989 return ret; 4990 } 4991 4992 return 0; 4993 } 4994 4995 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state, 4996 const struct intel_plane_state *plane_state) 4997 { 4998 enum plane_id plane_id = to_intel_plane(plane_state->uapi.plane)->id; 4999 int ret; 5000 5001 /* Watermarks calculated in master */ 5002 if (plane_state->planar_slave) 5003 return 0; 5004 5005 if (plane_state->planar_linked_plane) { 5006 const struct drm_framebuffer *fb = plane_state->hw.fb; 5007 enum plane_id y_plane_id = plane_state->planar_linked_plane->id; 5008 5009 WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state)); 5010 WARN_ON(!fb->format->is_yuv || 5011 fb->format->num_planes == 1); 5012 5013 ret = skl_build_plane_wm_single(crtc_state, plane_state, 5014 y_plane_id, 0); 5015 if (ret) 5016 return ret; 5017 5018 ret = skl_build_plane_wm_single(crtc_state, plane_state, 5019 plane_id, 1); 5020 if (ret) 5021 return ret; 5022 } else if (intel_wm_plane_visible(crtc_state, plane_state)) { 5023 ret = skl_build_plane_wm_single(crtc_state, plane_state, 5024 plane_id, 0); 5025 if (ret) 5026 return ret; 5027 } 5028 5029 return 0; 5030 } 5031 5032 static int skl_build_pipe_wm(struct intel_crtc_state *crtc_state) 5033 { 5034 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 5035 struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal; 5036 struct intel_plane *plane; 5037 const struct intel_plane_state *plane_state; 5038 int ret; 5039 5040 /* 5041 * We'll only calculate watermarks for planes that are actually 5042 * enabled, so make sure all other planes are set as disabled. 5043 */ 5044 memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes)); 5045 5046 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, 5047 crtc_state) { 5048 5049 if (INTEL_GEN(dev_priv) >= 11) 5050 ret = icl_build_plane_wm(crtc_state, plane_state); 5051 else 5052 ret = skl_build_plane_wm(crtc_state, plane_state); 5053 if (ret) 5054 return ret; 5055 } 5056 5057 pipe_wm->linetime = skl_compute_linetime_wm(crtc_state); 5058 5059 return 0; 5060 } 5061 5062 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, 5063 i915_reg_t reg, 5064 const struct skl_ddb_entry *entry) 5065 { 5066 if (entry->end) 5067 I915_WRITE_FW(reg, (entry->end - 1) << 16 | entry->start); 5068 else 5069 I915_WRITE_FW(reg, 0); 5070 } 5071 5072 static void skl_write_wm_level(struct drm_i915_private *dev_priv, 5073 i915_reg_t reg, 5074 const struct skl_wm_level *level) 5075 { 5076 u32 val = 0; 5077 5078 if (level->plane_en) 5079 val |= PLANE_WM_EN; 5080 if (level->ignore_lines) 5081 val |= PLANE_WM_IGNORE_LINES; 5082 val |= level->plane_res_b; 5083 val |= level->plane_res_l << PLANE_WM_LINES_SHIFT; 5084 5085 I915_WRITE_FW(reg, val); 5086 } 5087 5088 void skl_write_plane_wm(struct intel_plane *plane, 5089 const struct intel_crtc_state *crtc_state) 5090 { 5091 struct drm_i915_private *dev_priv = to_i915(plane->base.dev); 5092 int level, max_level = ilk_wm_max_level(dev_priv); 5093 enum plane_id plane_id = plane->id; 5094 enum pipe pipe = plane->pipe; 5095 const struct skl_plane_wm *wm = 5096 &crtc_state->wm.skl.optimal.planes[plane_id]; 5097 const struct skl_ddb_entry *ddb_y = 5098 &crtc_state->wm.skl.plane_ddb_y[plane_id]; 5099 const struct skl_ddb_entry *ddb_uv = 5100 &crtc_state->wm.skl.plane_ddb_uv[plane_id]; 5101 5102 for (level = 0; level <= max_level; level++) { 5103 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level), 5104 &wm->wm[level]); 5105 } 5106 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id), 5107 &wm->trans_wm); 5108 5109 if (INTEL_GEN(dev_priv) >= 11) { 5110 skl_ddb_entry_write(dev_priv, 5111 PLANE_BUF_CFG(pipe, plane_id), ddb_y); 5112 return; 5113 } 5114 5115 if (wm->is_planar) 5116 swap(ddb_y, ddb_uv); 5117 5118 skl_ddb_entry_write(dev_priv, 5119 PLANE_BUF_CFG(pipe, plane_id), ddb_y); 5120 skl_ddb_entry_write(dev_priv, 5121 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv); 5122 } 5123 5124 void skl_write_cursor_wm(struct intel_plane *plane, 5125 const struct intel_crtc_state *crtc_state) 5126 { 5127 struct drm_i915_private *dev_priv = to_i915(plane->base.dev); 5128 int level, max_level = ilk_wm_max_level(dev_priv); 5129 enum plane_id plane_id = plane->id; 5130 enum pipe pipe = plane->pipe; 5131 const struct skl_plane_wm *wm = 5132 &crtc_state->wm.skl.optimal.planes[plane_id]; 5133 const struct skl_ddb_entry *ddb = 5134 &crtc_state->wm.skl.plane_ddb_y[plane_id]; 5135 5136 for (level = 0; level <= max_level; level++) { 5137 skl_write_wm_level(dev_priv, CUR_WM(pipe, level), 5138 &wm->wm[level]); 5139 } 5140 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm); 5141 5142 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb); 5143 } 5144 5145 bool skl_wm_level_equals(const struct skl_wm_level *l1, 5146 const struct skl_wm_level *l2) 5147 { 5148 return l1->plane_en == l2->plane_en && 5149 l1->ignore_lines == l2->ignore_lines && 5150 l1->plane_res_l == l2->plane_res_l && 5151 l1->plane_res_b == l2->plane_res_b; 5152 } 5153 5154 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv, 5155 const struct skl_plane_wm *wm1, 5156 const struct skl_plane_wm *wm2) 5157 { 5158 int level, max_level = ilk_wm_max_level(dev_priv); 5159 5160 for (level = 0; level <= max_level; level++) { 5161 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]) || 5162 !skl_wm_level_equals(&wm1->uv_wm[level], &wm2->uv_wm[level])) 5163 return false; 5164 } 5165 5166 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm); 5167 } 5168 5169 static bool skl_pipe_wm_equals(struct intel_crtc *crtc, 5170 const struct skl_pipe_wm *wm1, 5171 const struct skl_pipe_wm *wm2) 5172 { 5173 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 5174 enum plane_id plane_id; 5175 5176 for_each_plane_id_on_crtc(crtc, plane_id) { 5177 if (!skl_plane_wm_equals(dev_priv, 5178 &wm1->planes[plane_id], 5179 &wm2->planes[plane_id])) 5180 return false; 5181 } 5182 5183 return wm1->linetime == wm2->linetime; 5184 } 5185 5186 static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a, 5187 const struct skl_ddb_entry *b) 5188 { 5189 return a->start < b->end && b->start < a->end; 5190 } 5191 5192 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb, 5193 const struct skl_ddb_entry *entries, 5194 int num_entries, int ignore_idx) 5195 { 5196 int i; 5197 5198 for (i = 0; i < num_entries; i++) { 5199 if (i != ignore_idx && 5200 skl_ddb_entries_overlap(ddb, &entries[i])) 5201 return true; 5202 } 5203 5204 return false; 5205 } 5206 5207 static int 5208 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state, 5209 struct intel_crtc_state *new_crtc_state) 5210 { 5211 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state); 5212 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); 5213 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 5214 struct intel_plane *plane; 5215 5216 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) { 5217 struct intel_plane_state *plane_state; 5218 enum plane_id plane_id = plane->id; 5219 5220 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id], 5221 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) && 5222 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id], 5223 &new_crtc_state->wm.skl.plane_ddb_uv[plane_id])) 5224 continue; 5225 5226 plane_state = intel_atomic_get_plane_state(state, plane); 5227 if (IS_ERR(plane_state)) 5228 return PTR_ERR(plane_state); 5229 5230 new_crtc_state->update_planes |= BIT(plane_id); 5231 } 5232 5233 return 0; 5234 } 5235 5236 static int 5237 skl_compute_ddb(struct intel_atomic_state *state) 5238 { 5239 const struct drm_i915_private *dev_priv = to_i915(state->base.dev); 5240 struct skl_ddb_allocation *ddb = &state->wm_results.ddb; 5241 struct intel_crtc_state *old_crtc_state; 5242 struct intel_crtc_state *new_crtc_state; 5243 struct intel_crtc *crtc; 5244 int ret, i; 5245 5246 memcpy(ddb, &dev_priv->wm.skl_hw.ddb, sizeof(*ddb)); 5247 5248 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, 5249 new_crtc_state, i) { 5250 ret = skl_allocate_pipe_ddb(new_crtc_state, ddb); 5251 if (ret) 5252 return ret; 5253 5254 ret = skl_ddb_add_affected_planes(old_crtc_state, 5255 new_crtc_state); 5256 if (ret) 5257 return ret; 5258 } 5259 5260 return 0; 5261 } 5262 5263 static char enast(bool enable) 5264 { 5265 return enable ? '*' : ' '; 5266 } 5267 5268 static void 5269 skl_print_wm_changes(struct intel_atomic_state *state) 5270 { 5271 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 5272 const struct intel_crtc_state *old_crtc_state; 5273 const struct intel_crtc_state *new_crtc_state; 5274 struct intel_plane *plane; 5275 struct intel_crtc *crtc; 5276 int i; 5277 5278 if (!drm_debug_enabled(DRM_UT_KMS)) 5279 return; 5280 5281 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, 5282 new_crtc_state, i) { 5283 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm; 5284 5285 old_pipe_wm = &old_crtc_state->wm.skl.optimal; 5286 new_pipe_wm = &new_crtc_state->wm.skl.optimal; 5287 5288 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) { 5289 enum plane_id plane_id = plane->id; 5290 const struct skl_ddb_entry *old, *new; 5291 5292 old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id]; 5293 new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id]; 5294 5295 if (skl_ddb_entry_equal(old, new)) 5296 continue; 5297 5298 drm_dbg_kms(&dev_priv->drm, 5299 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n", 5300 plane->base.base.id, plane->base.name, 5301 old->start, old->end, new->start, new->end, 5302 skl_ddb_entry_size(old), skl_ddb_entry_size(new)); 5303 } 5304 5305 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) { 5306 enum plane_id plane_id = plane->id; 5307 const struct skl_plane_wm *old_wm, *new_wm; 5308 5309 old_wm = &old_pipe_wm->planes[plane_id]; 5310 new_wm = &new_pipe_wm->planes[plane_id]; 5311 5312 if (skl_plane_wm_equals(dev_priv, old_wm, new_wm)) 5313 continue; 5314 5315 drm_dbg_kms(&dev_priv->drm, 5316 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm" 5317 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm\n", 5318 plane->base.base.id, plane->base.name, 5319 enast(old_wm->wm[0].plane_en), enast(old_wm->wm[1].plane_en), 5320 enast(old_wm->wm[2].plane_en), enast(old_wm->wm[3].plane_en), 5321 enast(old_wm->wm[4].plane_en), enast(old_wm->wm[5].plane_en), 5322 enast(old_wm->wm[6].plane_en), enast(old_wm->wm[7].plane_en), 5323 enast(old_wm->trans_wm.plane_en), 5324 enast(new_wm->wm[0].plane_en), enast(new_wm->wm[1].plane_en), 5325 enast(new_wm->wm[2].plane_en), enast(new_wm->wm[3].plane_en), 5326 enast(new_wm->wm[4].plane_en), enast(new_wm->wm[5].plane_en), 5327 enast(new_wm->wm[6].plane_en), enast(new_wm->wm[7].plane_en), 5328 enast(new_wm->trans_wm.plane_en)); 5329 5330 drm_dbg_kms(&dev_priv->drm, 5331 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d" 5332 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d\n", 5333 plane->base.base.id, plane->base.name, 5334 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].plane_res_l, 5335 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].plane_res_l, 5336 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].plane_res_l, 5337 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].plane_res_l, 5338 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].plane_res_l, 5339 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].plane_res_l, 5340 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].plane_res_l, 5341 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].plane_res_l, 5342 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.plane_res_l, 5343 5344 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].plane_res_l, 5345 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].plane_res_l, 5346 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].plane_res_l, 5347 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].plane_res_l, 5348 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].plane_res_l, 5349 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].plane_res_l, 5350 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].plane_res_l, 5351 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].plane_res_l, 5352 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.plane_res_l); 5353 5354 drm_dbg_kms(&dev_priv->drm, 5355 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d" 5356 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n", 5357 plane->base.base.id, plane->base.name, 5358 old_wm->wm[0].plane_res_b, old_wm->wm[1].plane_res_b, 5359 old_wm->wm[2].plane_res_b, old_wm->wm[3].plane_res_b, 5360 old_wm->wm[4].plane_res_b, old_wm->wm[5].plane_res_b, 5361 old_wm->wm[6].plane_res_b, old_wm->wm[7].plane_res_b, 5362 old_wm->trans_wm.plane_res_b, 5363 new_wm->wm[0].plane_res_b, new_wm->wm[1].plane_res_b, 5364 new_wm->wm[2].plane_res_b, new_wm->wm[3].plane_res_b, 5365 new_wm->wm[4].plane_res_b, new_wm->wm[5].plane_res_b, 5366 new_wm->wm[6].plane_res_b, new_wm->wm[7].plane_res_b, 5367 new_wm->trans_wm.plane_res_b); 5368 5369 drm_dbg_kms(&dev_priv->drm, 5370 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d" 5371 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n", 5372 plane->base.base.id, plane->base.name, 5373 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc, 5374 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc, 5375 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc, 5376 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc, 5377 old_wm->trans_wm.min_ddb_alloc, 5378 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc, 5379 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc, 5380 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc, 5381 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc, 5382 new_wm->trans_wm.min_ddb_alloc); 5383 } 5384 } 5385 } 5386 5387 static int intel_add_all_pipes(struct intel_atomic_state *state) 5388 { 5389 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 5390 struct intel_crtc *crtc; 5391 5392 for_each_intel_crtc(&dev_priv->drm, crtc) { 5393 struct intel_crtc_state *crtc_state; 5394 5395 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc); 5396 if (IS_ERR(crtc_state)) 5397 return PTR_ERR(crtc_state); 5398 } 5399 5400 return 0; 5401 } 5402 5403 static int 5404 skl_ddb_add_affected_pipes(struct intel_atomic_state *state) 5405 { 5406 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 5407 int ret; 5408 5409 /* 5410 * If this is our first atomic update following hardware readout, 5411 * we can't trust the DDB that the BIOS programmed for us. Let's 5412 * pretend that all pipes switched active status so that we'll 5413 * ensure a full DDB recompute. 5414 */ 5415 if (dev_priv->wm.distrust_bios_wm) { 5416 ret = drm_modeset_lock(&dev_priv->drm.mode_config.connection_mutex, 5417 state->base.acquire_ctx); 5418 if (ret) 5419 return ret; 5420 5421 state->active_pipe_changes = INTEL_INFO(dev_priv)->pipe_mask; 5422 5423 /* 5424 * We usually only initialize state->active_pipes if we 5425 * we're doing a modeset; make sure this field is always 5426 * initialized during the sanitization process that happens 5427 * on the first commit too. 5428 */ 5429 if (!state->modeset) 5430 state->active_pipes = dev_priv->active_pipes; 5431 } 5432 5433 /* 5434 * If the modeset changes which CRTC's are active, we need to 5435 * recompute the DDB allocation for *all* active pipes, even 5436 * those that weren't otherwise being modified in any way by this 5437 * atomic commit. Due to the shrinking of the per-pipe allocations 5438 * when new active CRTC's are added, it's possible for a pipe that 5439 * we were already using and aren't changing at all here to suddenly 5440 * become invalid if its DDB needs exceeds its new allocation. 5441 * 5442 * Note that if we wind up doing a full DDB recompute, we can't let 5443 * any other display updates race with this transaction, so we need 5444 * to grab the lock on *all* CRTC's. 5445 */ 5446 if (state->active_pipe_changes || state->modeset) { 5447 state->wm_results.dirty_pipes = INTEL_INFO(dev_priv)->pipe_mask; 5448 5449 ret = intel_add_all_pipes(state); 5450 if (ret) 5451 return ret; 5452 } 5453 5454 return 0; 5455 } 5456 5457 /* 5458 * To make sure the cursor watermark registers are always consistent 5459 * with our computed state the following scenario needs special 5460 * treatment: 5461 * 5462 * 1. enable cursor 5463 * 2. move cursor entirely offscreen 5464 * 3. disable cursor 5465 * 5466 * Step 2. does call .disable_plane() but does not zero the watermarks 5467 * (since we consider an offscreen cursor still active for the purposes 5468 * of watermarks). Step 3. would not normally call .disable_plane() 5469 * because the actual plane visibility isn't changing, and we don't 5470 * deallocate the cursor ddb until the pipe gets disabled. So we must 5471 * force step 3. to call .disable_plane() to update the watermark 5472 * registers properly. 5473 * 5474 * Other planes do not suffer from this issues as their watermarks are 5475 * calculated based on the actual plane visibility. The only time this 5476 * can trigger for the other planes is during the initial readout as the 5477 * default value of the watermarks registers is not zero. 5478 */ 5479 static int skl_wm_add_affected_planes(struct intel_atomic_state *state, 5480 struct intel_crtc *crtc) 5481 { 5482 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 5483 const struct intel_crtc_state *old_crtc_state = 5484 intel_atomic_get_old_crtc_state(state, crtc); 5485 struct intel_crtc_state *new_crtc_state = 5486 intel_atomic_get_new_crtc_state(state, crtc); 5487 struct intel_plane *plane; 5488 5489 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) { 5490 struct intel_plane_state *plane_state; 5491 enum plane_id plane_id = plane->id; 5492 5493 /* 5494 * Force a full wm update for every plane on modeset. 5495 * Required because the reset value of the wm registers 5496 * is non-zero, whereas we want all disabled planes to 5497 * have zero watermarks. So if we turn off the relevant 5498 * power well the hardware state will go out of sync 5499 * with the software state. 5500 */ 5501 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) && 5502 skl_plane_wm_equals(dev_priv, 5503 &old_crtc_state->wm.skl.optimal.planes[plane_id], 5504 &new_crtc_state->wm.skl.optimal.planes[plane_id])) 5505 continue; 5506 5507 plane_state = intel_atomic_get_plane_state(state, plane); 5508 if (IS_ERR(plane_state)) 5509 return PTR_ERR(plane_state); 5510 5511 new_crtc_state->update_planes |= BIT(plane_id); 5512 } 5513 5514 return 0; 5515 } 5516 5517 static int 5518 skl_compute_wm(struct intel_atomic_state *state) 5519 { 5520 struct intel_crtc *crtc; 5521 struct intel_crtc_state *new_crtc_state; 5522 struct intel_crtc_state *old_crtc_state; 5523 struct skl_ddb_values *results = &state->wm_results; 5524 int ret, i; 5525 5526 /* Clear all dirty flags */ 5527 results->dirty_pipes = 0; 5528 5529 ret = skl_ddb_add_affected_pipes(state); 5530 if (ret) 5531 return ret; 5532 5533 /* 5534 * Calculate WM's for all pipes that are part of this transaction. 5535 * Note that skl_ddb_add_affected_pipes may have added more CRTC's that 5536 * weren't otherwise being modified (and set bits in dirty_pipes) if 5537 * pipe allocations had to change. 5538 */ 5539 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, 5540 new_crtc_state, i) { 5541 ret = skl_build_pipe_wm(new_crtc_state); 5542 if (ret) 5543 return ret; 5544 5545 ret = skl_wm_add_affected_planes(state, crtc); 5546 if (ret) 5547 return ret; 5548 5549 if (!skl_pipe_wm_equals(crtc, 5550 &old_crtc_state->wm.skl.optimal, 5551 &new_crtc_state->wm.skl.optimal)) 5552 results->dirty_pipes |= BIT(crtc->pipe); 5553 } 5554 5555 ret = skl_compute_ddb(state); 5556 if (ret) 5557 return ret; 5558 5559 skl_print_wm_changes(state); 5560 5561 return 0; 5562 } 5563 5564 static void skl_atomic_update_crtc_wm(struct intel_atomic_state *state, 5565 struct intel_crtc *crtc) 5566 { 5567 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 5568 const struct intel_crtc_state *crtc_state = 5569 intel_atomic_get_new_crtc_state(state, crtc); 5570 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal; 5571 enum pipe pipe = crtc->pipe; 5572 5573 if ((state->wm_results.dirty_pipes & BIT(crtc->pipe)) == 0) 5574 return; 5575 5576 I915_WRITE(PIPE_WM_LINETIME(pipe), pipe_wm->linetime); 5577 } 5578 5579 static void skl_initial_wm(struct intel_atomic_state *state, 5580 struct intel_crtc *crtc) 5581 { 5582 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 5583 const struct intel_crtc_state *crtc_state = 5584 intel_atomic_get_new_crtc_state(state, crtc); 5585 struct skl_ddb_values *results = &state->wm_results; 5586 5587 if ((results->dirty_pipes & BIT(crtc->pipe)) == 0) 5588 return; 5589 5590 mutex_lock(&dev_priv->wm.wm_mutex); 5591 5592 if (crtc_state->uapi.active_changed) 5593 skl_atomic_update_crtc_wm(state, crtc); 5594 5595 mutex_unlock(&dev_priv->wm.wm_mutex); 5596 } 5597 5598 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv, 5599 struct intel_wm_config *config) 5600 { 5601 struct intel_crtc *crtc; 5602 5603 /* Compute the currently _active_ config */ 5604 for_each_intel_crtc(&dev_priv->drm, crtc) { 5605 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk; 5606 5607 if (!wm->pipe_enabled) 5608 continue; 5609 5610 config->sprites_enabled |= wm->sprites_enabled; 5611 config->sprites_scaled |= wm->sprites_scaled; 5612 config->num_pipes_active++; 5613 } 5614 } 5615 5616 static void ilk_program_watermarks(struct drm_i915_private *dev_priv) 5617 { 5618 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm; 5619 struct ilk_wm_maximums max; 5620 struct intel_wm_config config = {}; 5621 struct ilk_wm_values results = {}; 5622 enum intel_ddb_partitioning partitioning; 5623 5624 ilk_compute_wm_config(dev_priv, &config); 5625 5626 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max); 5627 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2); 5628 5629 /* 5/6 split only in single pipe config on IVB+ */ 5630 if (INTEL_GEN(dev_priv) >= 7 && 5631 config.num_pipes_active == 1 && config.sprites_enabled) { 5632 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max); 5633 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6); 5634 5635 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6); 5636 } else { 5637 best_lp_wm = &lp_wm_1_2; 5638 } 5639 5640 partitioning = (best_lp_wm == &lp_wm_1_2) ? 5641 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6; 5642 5643 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results); 5644 5645 ilk_write_wm_values(dev_priv, &results); 5646 } 5647 5648 static void ilk_initial_watermarks(struct intel_atomic_state *state, 5649 struct intel_crtc *crtc) 5650 { 5651 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 5652 const struct intel_crtc_state *crtc_state = 5653 intel_atomic_get_new_crtc_state(state, crtc); 5654 5655 mutex_lock(&dev_priv->wm.wm_mutex); 5656 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate; 5657 ilk_program_watermarks(dev_priv); 5658 mutex_unlock(&dev_priv->wm.wm_mutex); 5659 } 5660 5661 static void ilk_optimize_watermarks(struct intel_atomic_state *state, 5662 struct intel_crtc *crtc) 5663 { 5664 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 5665 const struct intel_crtc_state *crtc_state = 5666 intel_atomic_get_new_crtc_state(state, crtc); 5667 5668 if (!crtc_state->wm.need_postvbl_update) 5669 return; 5670 5671 mutex_lock(&dev_priv->wm.wm_mutex); 5672 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal; 5673 ilk_program_watermarks(dev_priv); 5674 mutex_unlock(&dev_priv->wm.wm_mutex); 5675 } 5676 5677 static inline void skl_wm_level_from_reg_val(u32 val, 5678 struct skl_wm_level *level) 5679 { 5680 level->plane_en = val & PLANE_WM_EN; 5681 level->ignore_lines = val & PLANE_WM_IGNORE_LINES; 5682 level->plane_res_b = val & PLANE_WM_BLOCKS_MASK; 5683 level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) & 5684 PLANE_WM_LINES_MASK; 5685 } 5686 5687 void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc, 5688 struct skl_pipe_wm *out) 5689 { 5690 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 5691 enum pipe pipe = crtc->pipe; 5692 int level, max_level; 5693 enum plane_id plane_id; 5694 u32 val; 5695 5696 max_level = ilk_wm_max_level(dev_priv); 5697 5698 for_each_plane_id_on_crtc(crtc, plane_id) { 5699 struct skl_plane_wm *wm = &out->planes[plane_id]; 5700 5701 for (level = 0; level <= max_level; level++) { 5702 if (plane_id != PLANE_CURSOR) 5703 val = I915_READ(PLANE_WM(pipe, plane_id, level)); 5704 else 5705 val = I915_READ(CUR_WM(pipe, level)); 5706 5707 skl_wm_level_from_reg_val(val, &wm->wm[level]); 5708 } 5709 5710 if (plane_id != PLANE_CURSOR) 5711 val = I915_READ(PLANE_WM_TRANS(pipe, plane_id)); 5712 else 5713 val = I915_READ(CUR_WM_TRANS(pipe)); 5714 5715 skl_wm_level_from_reg_val(val, &wm->trans_wm); 5716 } 5717 5718 if (!crtc->active) 5719 return; 5720 5721 out->linetime = I915_READ(PIPE_WM_LINETIME(pipe)); 5722 } 5723 5724 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv) 5725 { 5726 struct skl_ddb_values *hw = &dev_priv->wm.skl_hw; 5727 struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb; 5728 struct intel_crtc *crtc; 5729 struct intel_crtc_state *crtc_state; 5730 5731 skl_ddb_get_hw_state(dev_priv, ddb); 5732 for_each_intel_crtc(&dev_priv->drm, crtc) { 5733 crtc_state = to_intel_crtc_state(crtc->base.state); 5734 5735 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal); 5736 5737 if (crtc->active) 5738 hw->dirty_pipes |= BIT(crtc->pipe); 5739 } 5740 5741 if (dev_priv->active_pipes) { 5742 /* Fully recompute DDB on first atomic commit */ 5743 dev_priv->wm.distrust_bios_wm = true; 5744 } 5745 } 5746 5747 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc) 5748 { 5749 struct drm_device *dev = crtc->base.dev; 5750 struct drm_i915_private *dev_priv = to_i915(dev); 5751 struct ilk_wm_values *hw = &dev_priv->wm.hw; 5752 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); 5753 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal; 5754 enum pipe pipe = crtc->pipe; 5755 static const i915_reg_t wm0_pipe_reg[] = { 5756 [PIPE_A] = WM0_PIPEA_ILK, 5757 [PIPE_B] = WM0_PIPEB_ILK, 5758 [PIPE_C] = WM0_PIPEC_IVB, 5759 }; 5760 5761 hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]); 5762 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) 5763 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe)); 5764 5765 memset(active, 0, sizeof(*active)); 5766 5767 active->pipe_enabled = crtc->active; 5768 5769 if (active->pipe_enabled) { 5770 u32 tmp = hw->wm_pipe[pipe]; 5771 5772 /* 5773 * For active pipes LP0 watermark is marked as 5774 * enabled, and LP1+ watermaks as disabled since 5775 * we can't really reverse compute them in case 5776 * multiple pipes are active. 5777 */ 5778 active->wm[0].enable = true; 5779 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT; 5780 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT; 5781 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK; 5782 active->linetime = hw->wm_linetime[pipe]; 5783 } else { 5784 int level, max_level = ilk_wm_max_level(dev_priv); 5785 5786 /* 5787 * For inactive pipes, all watermark levels 5788 * should be marked as enabled but zeroed, 5789 * which is what we'd compute them to. 5790 */ 5791 for (level = 0; level <= max_level; level++) 5792 active->wm[level].enable = true; 5793 } 5794 5795 crtc->wm.active.ilk = *active; 5796 } 5797 5798 #define _FW_WM(value, plane) \ 5799 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT) 5800 #define _FW_WM_VLV(value, plane) \ 5801 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT) 5802 5803 static void g4x_read_wm_values(struct drm_i915_private *dev_priv, 5804 struct g4x_wm_values *wm) 5805 { 5806 u32 tmp; 5807 5808 tmp = I915_READ(DSPFW1); 5809 wm->sr.plane = _FW_WM(tmp, SR); 5810 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB); 5811 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB); 5812 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA); 5813 5814 tmp = I915_READ(DSPFW2); 5815 wm->fbc_en = tmp & DSPFW_FBC_SR_EN; 5816 wm->sr.fbc = _FW_WM(tmp, FBC_SR); 5817 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR); 5818 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB); 5819 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA); 5820 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA); 5821 5822 tmp = I915_READ(DSPFW3); 5823 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN; 5824 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR); 5825 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR); 5826 wm->hpll.plane = _FW_WM(tmp, HPLL_SR); 5827 } 5828 5829 static void vlv_read_wm_values(struct drm_i915_private *dev_priv, 5830 struct vlv_wm_values *wm) 5831 { 5832 enum pipe pipe; 5833 u32 tmp; 5834 5835 for_each_pipe(dev_priv, pipe) { 5836 tmp = I915_READ(VLV_DDL(pipe)); 5837 5838 wm->ddl[pipe].plane[PLANE_PRIMARY] = 5839 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); 5840 wm->ddl[pipe].plane[PLANE_CURSOR] = 5841 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); 5842 wm->ddl[pipe].plane[PLANE_SPRITE0] = 5843 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); 5844 wm->ddl[pipe].plane[PLANE_SPRITE1] = 5845 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); 5846 } 5847 5848 tmp = I915_READ(DSPFW1); 5849 wm->sr.plane = _FW_WM(tmp, SR); 5850 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB); 5851 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB); 5852 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA); 5853 5854 tmp = I915_READ(DSPFW2); 5855 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB); 5856 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA); 5857 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA); 5858 5859 tmp = I915_READ(DSPFW3); 5860 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR); 5861 5862 if (IS_CHERRYVIEW(dev_priv)) { 5863 tmp = I915_READ(DSPFW7_CHV); 5864 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED); 5865 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC); 5866 5867 tmp = I915_READ(DSPFW8_CHV); 5868 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF); 5869 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE); 5870 5871 tmp = I915_READ(DSPFW9_CHV); 5872 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC); 5873 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC); 5874 5875 tmp = I915_READ(DSPHOWM); 5876 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9; 5877 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8; 5878 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8; 5879 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8; 5880 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8; 5881 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8; 5882 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8; 5883 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8; 5884 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8; 5885 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8; 5886 } else { 5887 tmp = I915_READ(DSPFW7); 5888 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED); 5889 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC); 5890 5891 tmp = I915_READ(DSPHOWM); 5892 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9; 5893 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8; 5894 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8; 5895 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8; 5896 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8; 5897 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8; 5898 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8; 5899 } 5900 } 5901 5902 #undef _FW_WM 5903 #undef _FW_WM_VLV 5904 5905 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv) 5906 { 5907 struct g4x_wm_values *wm = &dev_priv->wm.g4x; 5908 struct intel_crtc *crtc; 5909 5910 g4x_read_wm_values(dev_priv, wm); 5911 5912 wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN; 5913 5914 for_each_intel_crtc(&dev_priv->drm, crtc) { 5915 struct intel_crtc_state *crtc_state = 5916 to_intel_crtc_state(crtc->base.state); 5917 struct g4x_wm_state *active = &crtc->wm.active.g4x; 5918 struct g4x_pipe_wm *raw; 5919 enum pipe pipe = crtc->pipe; 5920 enum plane_id plane_id; 5921 int level, max_level; 5922 5923 active->cxsr = wm->cxsr; 5924 active->hpll_en = wm->hpll_en; 5925 active->fbc_en = wm->fbc_en; 5926 5927 active->sr = wm->sr; 5928 active->hpll = wm->hpll; 5929 5930 for_each_plane_id_on_crtc(crtc, plane_id) { 5931 active->wm.plane[plane_id] = 5932 wm->pipe[pipe].plane[plane_id]; 5933 } 5934 5935 if (wm->cxsr && wm->hpll_en) 5936 max_level = G4X_WM_LEVEL_HPLL; 5937 else if (wm->cxsr) 5938 max_level = G4X_WM_LEVEL_SR; 5939 else 5940 max_level = G4X_WM_LEVEL_NORMAL; 5941 5942 level = G4X_WM_LEVEL_NORMAL; 5943 raw = &crtc_state->wm.g4x.raw[level]; 5944 for_each_plane_id_on_crtc(crtc, plane_id) 5945 raw->plane[plane_id] = active->wm.plane[plane_id]; 5946 5947 if (++level > max_level) 5948 goto out; 5949 5950 raw = &crtc_state->wm.g4x.raw[level]; 5951 raw->plane[PLANE_PRIMARY] = active->sr.plane; 5952 raw->plane[PLANE_CURSOR] = active->sr.cursor; 5953 raw->plane[PLANE_SPRITE0] = 0; 5954 raw->fbc = active->sr.fbc; 5955 5956 if (++level > max_level) 5957 goto out; 5958 5959 raw = &crtc_state->wm.g4x.raw[level]; 5960 raw->plane[PLANE_PRIMARY] = active->hpll.plane; 5961 raw->plane[PLANE_CURSOR] = active->hpll.cursor; 5962 raw->plane[PLANE_SPRITE0] = 0; 5963 raw->fbc = active->hpll.fbc; 5964 5965 out: 5966 for_each_plane_id_on_crtc(crtc, plane_id) 5967 g4x_raw_plane_wm_set(crtc_state, level, 5968 plane_id, USHRT_MAX); 5969 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX); 5970 5971 crtc_state->wm.g4x.optimal = *active; 5972 crtc_state->wm.g4x.intermediate = *active; 5973 5974 drm_dbg_kms(&dev_priv->drm, 5975 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n", 5976 pipe_name(pipe), 5977 wm->pipe[pipe].plane[PLANE_PRIMARY], 5978 wm->pipe[pipe].plane[PLANE_CURSOR], 5979 wm->pipe[pipe].plane[PLANE_SPRITE0]); 5980 } 5981 5982 drm_dbg_kms(&dev_priv->drm, 5983 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n", 5984 wm->sr.plane, wm->sr.cursor, wm->sr.fbc); 5985 drm_dbg_kms(&dev_priv->drm, 5986 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n", 5987 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc); 5988 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n", 5989 yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en)); 5990 } 5991 5992 void g4x_wm_sanitize(struct drm_i915_private *dev_priv) 5993 { 5994 struct intel_plane *plane; 5995 struct intel_crtc *crtc; 5996 5997 mutex_lock(&dev_priv->wm.wm_mutex); 5998 5999 for_each_intel_plane(&dev_priv->drm, plane) { 6000 struct intel_crtc *crtc = 6001 intel_get_crtc_for_pipe(dev_priv, plane->pipe); 6002 struct intel_crtc_state *crtc_state = 6003 to_intel_crtc_state(crtc->base.state); 6004 struct intel_plane_state *plane_state = 6005 to_intel_plane_state(plane->base.state); 6006 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal; 6007 enum plane_id plane_id = plane->id; 6008 int level; 6009 6010 if (plane_state->uapi.visible) 6011 continue; 6012 6013 for (level = 0; level < 3; level++) { 6014 struct g4x_pipe_wm *raw = 6015 &crtc_state->wm.g4x.raw[level]; 6016 6017 raw->plane[plane_id] = 0; 6018 wm_state->wm.plane[plane_id] = 0; 6019 } 6020 6021 if (plane_id == PLANE_PRIMARY) { 6022 for (level = 0; level < 3; level++) { 6023 struct g4x_pipe_wm *raw = 6024 &crtc_state->wm.g4x.raw[level]; 6025 raw->fbc = 0; 6026 } 6027 6028 wm_state->sr.fbc = 0; 6029 wm_state->hpll.fbc = 0; 6030 wm_state->fbc_en = false; 6031 } 6032 } 6033 6034 for_each_intel_crtc(&dev_priv->drm, crtc) { 6035 struct intel_crtc_state *crtc_state = 6036 to_intel_crtc_state(crtc->base.state); 6037 6038 crtc_state->wm.g4x.intermediate = 6039 crtc_state->wm.g4x.optimal; 6040 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal; 6041 } 6042 6043 g4x_program_watermarks(dev_priv); 6044 6045 mutex_unlock(&dev_priv->wm.wm_mutex); 6046 } 6047 6048 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv) 6049 { 6050 struct vlv_wm_values *wm = &dev_priv->wm.vlv; 6051 struct intel_crtc *crtc; 6052 u32 val; 6053 6054 vlv_read_wm_values(dev_priv, wm); 6055 6056 wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN; 6057 wm->level = VLV_WM_LEVEL_PM2; 6058 6059 if (IS_CHERRYVIEW(dev_priv)) { 6060 vlv_punit_get(dev_priv); 6061 6062 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM); 6063 if (val & DSP_MAXFIFO_PM5_ENABLE) 6064 wm->level = VLV_WM_LEVEL_PM5; 6065 6066 /* 6067 * If DDR DVFS is disabled in the BIOS, Punit 6068 * will never ack the request. So if that happens 6069 * assume we don't have to enable/disable DDR DVFS 6070 * dynamically. To test that just set the REQ_ACK 6071 * bit to poke the Punit, but don't change the 6072 * HIGH/LOW bits so that we don't actually change 6073 * the current state. 6074 */ 6075 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2); 6076 val |= FORCE_DDR_FREQ_REQ_ACK; 6077 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val); 6078 6079 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) & 6080 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) { 6081 drm_dbg_kms(&dev_priv->drm, 6082 "Punit not acking DDR DVFS request, " 6083 "assuming DDR DVFS is disabled\n"); 6084 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5; 6085 } else { 6086 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2); 6087 if ((val & FORCE_DDR_HIGH_FREQ) == 0) 6088 wm->level = VLV_WM_LEVEL_DDR_DVFS; 6089 } 6090 6091 vlv_punit_put(dev_priv); 6092 } 6093 6094 for_each_intel_crtc(&dev_priv->drm, crtc) { 6095 struct intel_crtc_state *crtc_state = 6096 to_intel_crtc_state(crtc->base.state); 6097 struct vlv_wm_state *active = &crtc->wm.active.vlv; 6098 const struct vlv_fifo_state *fifo_state = 6099 &crtc_state->wm.vlv.fifo_state; 6100 enum pipe pipe = crtc->pipe; 6101 enum plane_id plane_id; 6102 int level; 6103 6104 vlv_get_fifo_size(crtc_state); 6105 6106 active->num_levels = wm->level + 1; 6107 active->cxsr = wm->cxsr; 6108 6109 for (level = 0; level < active->num_levels; level++) { 6110 struct g4x_pipe_wm *raw = 6111 &crtc_state->wm.vlv.raw[level]; 6112 6113 active->sr[level].plane = wm->sr.plane; 6114 active->sr[level].cursor = wm->sr.cursor; 6115 6116 for_each_plane_id_on_crtc(crtc, plane_id) { 6117 active->wm[level].plane[plane_id] = 6118 wm->pipe[pipe].plane[plane_id]; 6119 6120 raw->plane[plane_id] = 6121 vlv_invert_wm_value(active->wm[level].plane[plane_id], 6122 fifo_state->plane[plane_id]); 6123 } 6124 } 6125 6126 for_each_plane_id_on_crtc(crtc, plane_id) 6127 vlv_raw_plane_wm_set(crtc_state, level, 6128 plane_id, USHRT_MAX); 6129 vlv_invalidate_wms(crtc, active, level); 6130 6131 crtc_state->wm.vlv.optimal = *active; 6132 crtc_state->wm.vlv.intermediate = *active; 6133 6134 drm_dbg_kms(&dev_priv->drm, 6135 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n", 6136 pipe_name(pipe), 6137 wm->pipe[pipe].plane[PLANE_PRIMARY], 6138 wm->pipe[pipe].plane[PLANE_CURSOR], 6139 wm->pipe[pipe].plane[PLANE_SPRITE0], 6140 wm->pipe[pipe].plane[PLANE_SPRITE1]); 6141 } 6142 6143 drm_dbg_kms(&dev_priv->drm, 6144 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n", 6145 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr); 6146 } 6147 6148 void vlv_wm_sanitize(struct drm_i915_private *dev_priv) 6149 { 6150 struct intel_plane *plane; 6151 struct intel_crtc *crtc; 6152 6153 mutex_lock(&dev_priv->wm.wm_mutex); 6154 6155 for_each_intel_plane(&dev_priv->drm, plane) { 6156 struct intel_crtc *crtc = 6157 intel_get_crtc_for_pipe(dev_priv, plane->pipe); 6158 struct intel_crtc_state *crtc_state = 6159 to_intel_crtc_state(crtc->base.state); 6160 struct intel_plane_state *plane_state = 6161 to_intel_plane_state(plane->base.state); 6162 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal; 6163 const struct vlv_fifo_state *fifo_state = 6164 &crtc_state->wm.vlv.fifo_state; 6165 enum plane_id plane_id = plane->id; 6166 int level; 6167 6168 if (plane_state->uapi.visible) 6169 continue; 6170 6171 for (level = 0; level < wm_state->num_levels; level++) { 6172 struct g4x_pipe_wm *raw = 6173 &crtc_state->wm.vlv.raw[level]; 6174 6175 raw->plane[plane_id] = 0; 6176 6177 wm_state->wm[level].plane[plane_id] = 6178 vlv_invert_wm_value(raw->plane[plane_id], 6179 fifo_state->plane[plane_id]); 6180 } 6181 } 6182 6183 for_each_intel_crtc(&dev_priv->drm, crtc) { 6184 struct intel_crtc_state *crtc_state = 6185 to_intel_crtc_state(crtc->base.state); 6186 6187 crtc_state->wm.vlv.intermediate = 6188 crtc_state->wm.vlv.optimal; 6189 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal; 6190 } 6191 6192 vlv_program_watermarks(dev_priv); 6193 6194 mutex_unlock(&dev_priv->wm.wm_mutex); 6195 } 6196 6197 /* 6198 * FIXME should probably kill this and improve 6199 * the real watermark readout/sanitation instead 6200 */ 6201 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv) 6202 { 6203 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN); 6204 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN); 6205 I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN); 6206 6207 /* 6208 * Don't touch WM1S_LP_EN here. 6209 * Doing so could cause underruns. 6210 */ 6211 } 6212 6213 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv) 6214 { 6215 struct ilk_wm_values *hw = &dev_priv->wm.hw; 6216 struct intel_crtc *crtc; 6217 6218 ilk_init_lp_watermarks(dev_priv); 6219 6220 for_each_intel_crtc(&dev_priv->drm, crtc) 6221 ilk_pipe_wm_get_hw_state(crtc); 6222 6223 hw->wm_lp[0] = I915_READ(WM1_LP_ILK); 6224 hw->wm_lp[1] = I915_READ(WM2_LP_ILK); 6225 hw->wm_lp[2] = I915_READ(WM3_LP_ILK); 6226 6227 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK); 6228 if (INTEL_GEN(dev_priv) >= 7) { 6229 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB); 6230 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB); 6231 } 6232 6233 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) 6234 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ? 6235 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2; 6236 else if (IS_IVYBRIDGE(dev_priv)) 6237 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ? 6238 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2; 6239 6240 hw->enable_fbc_wm = 6241 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS); 6242 } 6243 6244 /** 6245 * intel_update_watermarks - update FIFO watermark values based on current modes 6246 * @crtc: the #intel_crtc on which to compute the WM 6247 * 6248 * Calculate watermark values for the various WM regs based on current mode 6249 * and plane configuration. 6250 * 6251 * There are several cases to deal with here: 6252 * - normal (i.e. non-self-refresh) 6253 * - self-refresh (SR) mode 6254 * - lines are large relative to FIFO size (buffer can hold up to 2) 6255 * - lines are small relative to FIFO size (buffer can hold more than 2 6256 * lines), so need to account for TLB latency 6257 * 6258 * The normal calculation is: 6259 * watermark = dotclock * bytes per pixel * latency 6260 * where latency is platform & configuration dependent (we assume pessimal 6261 * values here). 6262 * 6263 * The SR calculation is: 6264 * watermark = (trunc(latency/line time)+1) * surface width * 6265 * bytes per pixel 6266 * where 6267 * line time = htotal / dotclock 6268 * surface width = hdisplay for normal plane and 64 for cursor 6269 * and latency is assumed to be high, as above. 6270 * 6271 * The final value programmed to the register should always be rounded up, 6272 * and include an extra 2 entries to account for clock crossings. 6273 * 6274 * We don't use the sprite, so we can ignore that. And on Crestline we have 6275 * to set the non-SR watermarks to 8. 6276 */ 6277 void intel_update_watermarks(struct intel_crtc *crtc) 6278 { 6279 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 6280 6281 if (dev_priv->display.update_wm) 6282 dev_priv->display.update_wm(crtc); 6283 } 6284 6285 void intel_enable_ipc(struct drm_i915_private *dev_priv) 6286 { 6287 u32 val; 6288 6289 if (!HAS_IPC(dev_priv)) 6290 return; 6291 6292 val = I915_READ(DISP_ARB_CTL2); 6293 6294 if (dev_priv->ipc_enabled) 6295 val |= DISP_IPC_ENABLE; 6296 else 6297 val &= ~DISP_IPC_ENABLE; 6298 6299 I915_WRITE(DISP_ARB_CTL2, val); 6300 } 6301 6302 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv) 6303 { 6304 /* Display WA #0477 WaDisableIPC: skl */ 6305 if (IS_SKYLAKE(dev_priv)) 6306 return false; 6307 6308 /* Display WA #1141: SKL:all KBL:all CFL */ 6309 if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) 6310 return dev_priv->dram_info.symmetric_memory; 6311 6312 return true; 6313 } 6314 6315 void intel_init_ipc(struct drm_i915_private *dev_priv) 6316 { 6317 if (!HAS_IPC(dev_priv)) 6318 return; 6319 6320 dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv); 6321 6322 intel_enable_ipc(dev_priv); 6323 } 6324 6325 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv) 6326 { 6327 /* 6328 * On Ibex Peak and Cougar Point, we need to disable clock 6329 * gating for the panel power sequencer or it will fail to 6330 * start up when no ports are active. 6331 */ 6332 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE); 6333 } 6334 6335 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv) 6336 { 6337 enum pipe pipe; 6338 6339 for_each_pipe(dev_priv, pipe) { 6340 I915_WRITE(DSPCNTR(pipe), 6341 I915_READ(DSPCNTR(pipe)) | 6342 DISPPLANE_TRICKLE_FEED_DISABLE); 6343 6344 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe))); 6345 POSTING_READ(DSPSURF(pipe)); 6346 } 6347 } 6348 6349 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv) 6350 { 6351 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; 6352 6353 /* 6354 * Required for FBC 6355 * WaFbcDisableDpfcClockGating:ilk 6356 */ 6357 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE | 6358 ILK_DPFCUNIT_CLOCK_GATE_DISABLE | 6359 ILK_DPFDUNIT_CLOCK_GATE_ENABLE; 6360 6361 I915_WRITE(PCH_3DCGDIS0, 6362 MARIUNIT_CLOCK_GATE_DISABLE | 6363 SVSMUNIT_CLOCK_GATE_DISABLE); 6364 I915_WRITE(PCH_3DCGDIS1, 6365 VFMUNIT_CLOCK_GATE_DISABLE); 6366 6367 /* 6368 * According to the spec the following bits should be set in 6369 * order to enable memory self-refresh 6370 * The bit 22/21 of 0x42004 6371 * The bit 5 of 0x42020 6372 * The bit 15 of 0x45000 6373 */ 6374 I915_WRITE(ILK_DISPLAY_CHICKEN2, 6375 (I915_READ(ILK_DISPLAY_CHICKEN2) | 6376 ILK_DPARB_GATE | ILK_VSDPFD_FULL)); 6377 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE; 6378 I915_WRITE(DISP_ARB_CTL, 6379 (I915_READ(DISP_ARB_CTL) | 6380 DISP_FBC_WM_DIS)); 6381 6382 /* 6383 * Based on the document from hardware guys the following bits 6384 * should be set unconditionally in order to enable FBC. 6385 * The bit 22 of 0x42000 6386 * The bit 22 of 0x42004 6387 * The bit 7,8,9 of 0x42020. 6388 */ 6389 if (IS_IRONLAKE_M(dev_priv)) { 6390 /* WaFbcAsynchFlipDisableFbcQueue:ilk */ 6391 I915_WRITE(ILK_DISPLAY_CHICKEN1, 6392 I915_READ(ILK_DISPLAY_CHICKEN1) | 6393 ILK_FBCQ_DIS); 6394 I915_WRITE(ILK_DISPLAY_CHICKEN2, 6395 I915_READ(ILK_DISPLAY_CHICKEN2) | 6396 ILK_DPARB_GATE); 6397 } 6398 6399 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); 6400 6401 I915_WRITE(ILK_DISPLAY_CHICKEN2, 6402 I915_READ(ILK_DISPLAY_CHICKEN2) | 6403 ILK_ELPIN_409_SELECT); 6404 I915_WRITE(_3D_CHICKEN2, 6405 _3D_CHICKEN2_WM_READ_PIPELINED << 16 | 6406 _3D_CHICKEN2_WM_READ_PIPELINED); 6407 6408 /* WaDisableRenderCachePipelinedFlush:ilk */ 6409 I915_WRITE(CACHE_MODE_0, 6410 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE)); 6411 6412 /* WaDisable_RenderCache_OperationalFlush:ilk */ 6413 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 6414 6415 g4x_disable_trickle_feed(dev_priv); 6416 6417 ibx_init_clock_gating(dev_priv); 6418 } 6419 6420 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv) 6421 { 6422 enum pipe pipe; 6423 u32 val; 6424 6425 /* 6426 * On Ibex Peak and Cougar Point, we need to disable clock 6427 * gating for the panel power sequencer or it will fail to 6428 * start up when no ports are active. 6429 */ 6430 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE | 6431 PCH_DPLUNIT_CLOCK_GATE_DISABLE | 6432 PCH_CPUNIT_CLOCK_GATE_DISABLE); 6433 I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) | 6434 DPLS_EDP_PPS_FIX_DIS); 6435 /* The below fixes the weird display corruption, a few pixels shifted 6436 * downward, on (only) LVDS of some HP laptops with IVY. 6437 */ 6438 for_each_pipe(dev_priv, pipe) { 6439 val = I915_READ(TRANS_CHICKEN2(pipe)); 6440 val |= TRANS_CHICKEN2_TIMING_OVERRIDE; 6441 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED; 6442 if (dev_priv->vbt.fdi_rx_polarity_inverted) 6443 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED; 6444 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER; 6445 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH; 6446 I915_WRITE(TRANS_CHICKEN2(pipe), val); 6447 } 6448 /* WADP0ClockGatingDisable */ 6449 for_each_pipe(dev_priv, pipe) { 6450 I915_WRITE(TRANS_CHICKEN1(pipe), 6451 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); 6452 } 6453 } 6454 6455 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv) 6456 { 6457 u32 tmp; 6458 6459 tmp = I915_READ(MCH_SSKPD); 6460 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) 6461 drm_dbg_kms(&dev_priv->drm, 6462 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n", 6463 tmp); 6464 } 6465 6466 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv) 6467 { 6468 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; 6469 6470 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); 6471 6472 I915_WRITE(ILK_DISPLAY_CHICKEN2, 6473 I915_READ(ILK_DISPLAY_CHICKEN2) | 6474 ILK_ELPIN_409_SELECT); 6475 6476 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */ 6477 I915_WRITE(_3D_CHICKEN, 6478 _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB)); 6479 6480 /* WaDisable_RenderCache_OperationalFlush:snb */ 6481 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 6482 6483 /* 6484 * BSpec recoomends 8x4 when MSAA is used, 6485 * however in practice 16x4 seems fastest. 6486 * 6487 * Note that PS/WM thread counts depend on the WIZ hashing 6488 * disable bit, which we don't touch here, but it's good 6489 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 6490 */ 6491 I915_WRITE(GEN6_GT_MODE, 6492 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); 6493 6494 I915_WRITE(CACHE_MODE_0, 6495 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); 6496 6497 I915_WRITE(GEN6_UCGCTL1, 6498 I915_READ(GEN6_UCGCTL1) | 6499 GEN6_BLBUNIT_CLOCK_GATE_DISABLE | 6500 GEN6_CSUNIT_CLOCK_GATE_DISABLE); 6501 6502 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock 6503 * gating disable must be set. Failure to set it results in 6504 * flickering pixels due to Z write ordering failures after 6505 * some amount of runtime in the Mesa "fire" demo, and Unigine 6506 * Sanctuary and Tropics, and apparently anything else with 6507 * alpha test or pixel discard. 6508 * 6509 * According to the spec, bit 11 (RCCUNIT) must also be set, 6510 * but we didn't debug actual testcases to find it out. 6511 * 6512 * WaDisableRCCUnitClockGating:snb 6513 * WaDisableRCPBUnitClockGating:snb 6514 */ 6515 I915_WRITE(GEN6_UCGCTL2, 6516 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE | 6517 GEN6_RCCUNIT_CLOCK_GATE_DISABLE); 6518 6519 /* WaStripsFansDisableFastClipPerformanceFix:snb */ 6520 I915_WRITE(_3D_CHICKEN3, 6521 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL)); 6522 6523 /* 6524 * Bspec says: 6525 * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and 6526 * 3DSTATE_SF number of SF output attributes is more than 16." 6527 */ 6528 I915_WRITE(_3D_CHICKEN3, 6529 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH)); 6530 6531 /* 6532 * According to the spec the following bits should be 6533 * set in order to enable memory self-refresh and fbc: 6534 * The bit21 and bit22 of 0x42000 6535 * The bit21 and bit22 of 0x42004 6536 * The bit5 and bit7 of 0x42020 6537 * The bit14 of 0x70180 6538 * The bit14 of 0x71180 6539 * 6540 * WaFbcAsynchFlipDisableFbcQueue:snb 6541 */ 6542 I915_WRITE(ILK_DISPLAY_CHICKEN1, 6543 I915_READ(ILK_DISPLAY_CHICKEN1) | 6544 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS); 6545 I915_WRITE(ILK_DISPLAY_CHICKEN2, 6546 I915_READ(ILK_DISPLAY_CHICKEN2) | 6547 ILK_DPARB_GATE | ILK_VSDPFD_FULL); 6548 I915_WRITE(ILK_DSPCLK_GATE_D, 6549 I915_READ(ILK_DSPCLK_GATE_D) | 6550 ILK_DPARBUNIT_CLOCK_GATE_ENABLE | 6551 ILK_DPFDUNIT_CLOCK_GATE_ENABLE); 6552 6553 g4x_disable_trickle_feed(dev_priv); 6554 6555 cpt_init_clock_gating(dev_priv); 6556 6557 gen6_check_mch_setup(dev_priv); 6558 } 6559 6560 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) 6561 { 6562 u32 reg = I915_READ(GEN7_FF_THREAD_MODE); 6563 6564 /* 6565 * WaVSThreadDispatchOverride:ivb,vlv 6566 * 6567 * This actually overrides the dispatch 6568 * mode for all thread types. 6569 */ 6570 reg &= ~GEN7_FF_SCHED_MASK; 6571 reg |= GEN7_FF_TS_SCHED_HW; 6572 reg |= GEN7_FF_VS_SCHED_HW; 6573 reg |= GEN7_FF_DS_SCHED_HW; 6574 6575 I915_WRITE(GEN7_FF_THREAD_MODE, reg); 6576 } 6577 6578 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv) 6579 { 6580 /* 6581 * TODO: this bit should only be enabled when really needed, then 6582 * disabled when not needed anymore in order to save power. 6583 */ 6584 if (HAS_PCH_LPT_LP(dev_priv)) 6585 I915_WRITE(SOUTH_DSPCLK_GATE_D, 6586 I915_READ(SOUTH_DSPCLK_GATE_D) | 6587 PCH_LP_PARTITION_LEVEL_DISABLE); 6588 6589 /* WADPOClockGatingDisable:hsw */ 6590 I915_WRITE(TRANS_CHICKEN1(PIPE_A), 6591 I915_READ(TRANS_CHICKEN1(PIPE_A)) | 6592 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); 6593 } 6594 6595 static void lpt_suspend_hw(struct drm_i915_private *dev_priv) 6596 { 6597 if (HAS_PCH_LPT_LP(dev_priv)) { 6598 u32 val = I915_READ(SOUTH_DSPCLK_GATE_D); 6599 6600 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE; 6601 I915_WRITE(SOUTH_DSPCLK_GATE_D, val); 6602 } 6603 } 6604 6605 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv, 6606 int general_prio_credits, 6607 int high_prio_credits) 6608 { 6609 u32 misccpctl; 6610 u32 val; 6611 6612 /* WaTempDisableDOPClkGating:bdw */ 6613 misccpctl = I915_READ(GEN7_MISCCPCTL); 6614 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE); 6615 6616 val = I915_READ(GEN8_L3SQCREG1); 6617 val &= ~L3_PRIO_CREDITS_MASK; 6618 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits); 6619 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits); 6620 I915_WRITE(GEN8_L3SQCREG1, val); 6621 6622 /* 6623 * Wait at least 100 clocks before re-enabling clock gating. 6624 * See the definition of L3SQCREG1 in BSpec. 6625 */ 6626 POSTING_READ(GEN8_L3SQCREG1); 6627 udelay(1); 6628 I915_WRITE(GEN7_MISCCPCTL, misccpctl); 6629 } 6630 6631 static void icl_init_clock_gating(struct drm_i915_private *dev_priv) 6632 { 6633 /* This is not an Wa. Enable to reduce Sampler power */ 6634 I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN, 6635 I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE); 6636 6637 /* WaEnable32PlaneMode:icl */ 6638 I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, 6639 _MASKED_BIT_ENABLE(GEN11_ENABLE_32_PLANE_MODE)); 6640 6641 /* 6642 * Wa_1408615072:icl,ehl (vsunit) 6643 * Wa_1407596294:icl,ehl (hsunit) 6644 */ 6645 intel_uncore_rmw(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE, 6646 0, VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS); 6647 6648 /* Wa_1407352427:icl,ehl */ 6649 intel_uncore_rmw(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE2, 6650 0, PSDUNIT_CLKGATE_DIS); 6651 } 6652 6653 static void tgl_init_clock_gating(struct drm_i915_private *dev_priv) 6654 { 6655 u32 vd_pg_enable = 0; 6656 unsigned int i; 6657 6658 /* Wa_1408615072:tgl */ 6659 intel_uncore_rmw(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE2, 6660 0, VSUNIT_CLKGATE_DIS_TGL); 6661 6662 /* This is not a WA. Enable VD HCP & MFX_ENC powergate */ 6663 for (i = 0; i < I915_MAX_VCS; i++) { 6664 if (HAS_ENGINE(dev_priv, _VCS(i))) 6665 vd_pg_enable |= VDN_HCP_POWERGATE_ENABLE(i) | 6666 VDN_MFX_POWERGATE_ENABLE(i); 6667 } 6668 6669 I915_WRITE(POWERGATE_ENABLE, 6670 I915_READ(POWERGATE_ENABLE) | vd_pg_enable); 6671 } 6672 6673 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv) 6674 { 6675 if (!HAS_PCH_CNP(dev_priv)) 6676 return; 6677 6678 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */ 6679 I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) | 6680 CNP_PWM_CGE_GATING_DISABLE); 6681 } 6682 6683 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv) 6684 { 6685 u32 val; 6686 cnp_init_clock_gating(dev_priv); 6687 6688 /* This is not an Wa. Enable for better image quality */ 6689 I915_WRITE(_3D_CHICKEN3, 6690 _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE)); 6691 6692 /* WaEnableChickenDCPR:cnl */ 6693 I915_WRITE(GEN8_CHICKEN_DCPR_1, 6694 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM); 6695 6696 /* WaFbcWakeMemOn:cnl */ 6697 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 6698 DISP_FBC_MEMORY_WAKE); 6699 6700 val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE); 6701 /* ReadHitWriteOnlyDisable:cnl */ 6702 val |= RCCUNIT_CLKGATE_DIS; 6703 /* WaSarbUnitClockGatingDisable:cnl (pre-prod) */ 6704 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0)) 6705 val |= SARBUNIT_CLKGATE_DIS; 6706 I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val); 6707 6708 /* Wa_2201832410:cnl */ 6709 val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE); 6710 val |= GWUNIT_CLKGATE_DIS; 6711 I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val); 6712 6713 /* WaDisableVFclkgate:cnl */ 6714 /* WaVFUnitClockGatingDisable:cnl */ 6715 val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE); 6716 val |= VFUNIT_CLKGATE_DIS; 6717 I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val); 6718 } 6719 6720 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv) 6721 { 6722 cnp_init_clock_gating(dev_priv); 6723 gen9_init_clock_gating(dev_priv); 6724 6725 /* WaFbcNukeOnHostModify:cfl */ 6726 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | 6727 ILK_DPFC_NUKE_ON_ANY_MODIFICATION); 6728 } 6729 6730 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv) 6731 { 6732 gen9_init_clock_gating(dev_priv); 6733 6734 /* WaDisableSDEUnitClockGating:kbl */ 6735 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0)) 6736 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | 6737 GEN8_SDEUNIT_CLOCK_GATE_DISABLE); 6738 6739 /* WaDisableGamClockGating:kbl */ 6740 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0)) 6741 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) | 6742 GEN6_GAMUNIT_CLOCK_GATE_DISABLE); 6743 6744 /* WaFbcNukeOnHostModify:kbl */ 6745 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | 6746 ILK_DPFC_NUKE_ON_ANY_MODIFICATION); 6747 } 6748 6749 static void skl_init_clock_gating(struct drm_i915_private *dev_priv) 6750 { 6751 gen9_init_clock_gating(dev_priv); 6752 6753 /* WAC6entrylatency:skl */ 6754 I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) | 6755 FBC_LLC_FULLY_OPEN); 6756 6757 /* WaFbcNukeOnHostModify:skl */ 6758 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | 6759 ILK_DPFC_NUKE_ON_ANY_MODIFICATION); 6760 } 6761 6762 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv) 6763 { 6764 enum pipe pipe; 6765 6766 /* WaSwitchSolVfFArbitrationPriority:bdw */ 6767 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); 6768 6769 /* WaPsrDPAMaskVBlankInSRD:bdw */ 6770 I915_WRITE(CHICKEN_PAR1_1, 6771 I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD); 6772 6773 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */ 6774 for_each_pipe(dev_priv, pipe) { 6775 I915_WRITE(CHICKEN_PIPESL_1(pipe), 6776 I915_READ(CHICKEN_PIPESL_1(pipe)) | 6777 BDW_DPRS_MASK_VBLANK_SRD); 6778 } 6779 6780 /* WaVSRefCountFullforceMissDisable:bdw */ 6781 /* WaDSRefCountFullforceMissDisable:bdw */ 6782 I915_WRITE(GEN7_FF_THREAD_MODE, 6783 I915_READ(GEN7_FF_THREAD_MODE) & 6784 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME)); 6785 6786 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL, 6787 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE)); 6788 6789 /* WaDisableSDEUnitClockGating:bdw */ 6790 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | 6791 GEN8_SDEUNIT_CLOCK_GATE_DISABLE); 6792 6793 /* WaProgramL3SqcReg1Default:bdw */ 6794 gen8_set_l3sqc_credits(dev_priv, 30, 2); 6795 6796 /* WaKVMNotificationOnConfigChange:bdw */ 6797 I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1) 6798 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT); 6799 6800 lpt_init_clock_gating(dev_priv); 6801 6802 /* WaDisableDopClockGating:bdw 6803 * 6804 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP 6805 * clock gating. 6806 */ 6807 I915_WRITE(GEN6_UCGCTL1, 6808 I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE); 6809 } 6810 6811 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv) 6812 { 6813 /* L3 caching of data atomics doesn't work -- disable it. */ 6814 I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE); 6815 I915_WRITE(HSW_ROW_CHICKEN3, 6816 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE)); 6817 6818 /* This is required by WaCatErrorRejectionIssue:hsw */ 6819 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, 6820 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | 6821 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); 6822 6823 /* WaVSRefCountFullforceMissDisable:hsw */ 6824 I915_WRITE(GEN7_FF_THREAD_MODE, 6825 I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME); 6826 6827 /* WaDisable_RenderCache_OperationalFlush:hsw */ 6828 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 6829 6830 /* enable HiZ Raw Stall Optimization */ 6831 I915_WRITE(CACHE_MODE_0_GEN7, 6832 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE)); 6833 6834 /* WaDisable4x2SubspanOptimization:hsw */ 6835 I915_WRITE(CACHE_MODE_1, 6836 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); 6837 6838 /* 6839 * BSpec recommends 8x4 when MSAA is used, 6840 * however in practice 16x4 seems fastest. 6841 * 6842 * Note that PS/WM thread counts depend on the WIZ hashing 6843 * disable bit, which we don't touch here, but it's good 6844 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 6845 */ 6846 I915_WRITE(GEN7_GT_MODE, 6847 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); 6848 6849 /* WaSampleCChickenBitEnable:hsw */ 6850 I915_WRITE(HALF_SLICE_CHICKEN3, 6851 _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE)); 6852 6853 /* WaSwitchSolVfFArbitrationPriority:hsw */ 6854 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); 6855 6856 lpt_init_clock_gating(dev_priv); 6857 } 6858 6859 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv) 6860 { 6861 u32 snpcr; 6862 6863 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE); 6864 6865 /* WaDisableEarlyCull:ivb */ 6866 I915_WRITE(_3D_CHICKEN3, 6867 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL)); 6868 6869 /* WaDisableBackToBackFlipFix:ivb */ 6870 I915_WRITE(IVB_CHICKEN3, 6871 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE | 6872 CHICKEN3_DGMG_DONE_FIX_DISABLE); 6873 6874 /* WaDisablePSDDualDispatchEnable:ivb */ 6875 if (IS_IVB_GT1(dev_priv)) 6876 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, 6877 _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); 6878 6879 /* WaDisable_RenderCache_OperationalFlush:ivb */ 6880 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 6881 6882 /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */ 6883 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1, 6884 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC); 6885 6886 /* WaApplyL3ControlAndL3ChickenMode:ivb */ 6887 I915_WRITE(GEN7_L3CNTLREG1, 6888 GEN7_WA_FOR_GEN7_L3_CONTROL); 6889 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, 6890 GEN7_WA_L3_CHICKEN_MODE); 6891 if (IS_IVB_GT1(dev_priv)) 6892 I915_WRITE(GEN7_ROW_CHICKEN2, 6893 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); 6894 else { 6895 /* must write both registers */ 6896 I915_WRITE(GEN7_ROW_CHICKEN2, 6897 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); 6898 I915_WRITE(GEN7_ROW_CHICKEN2_GT2, 6899 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); 6900 } 6901 6902 /* WaForceL3Serialization:ivb */ 6903 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) & 6904 ~L3SQ_URB_READ_CAM_MATCH_DISABLE); 6905 6906 /* 6907 * According to the spec, bit 13 (RCZUNIT) must be set on IVB. 6908 * This implements the WaDisableRCZUnitClockGating:ivb workaround. 6909 */ 6910 I915_WRITE(GEN6_UCGCTL2, 6911 GEN6_RCZUNIT_CLOCK_GATE_DISABLE); 6912 6913 /* This is required by WaCatErrorRejectionIssue:ivb */ 6914 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, 6915 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | 6916 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); 6917 6918 g4x_disable_trickle_feed(dev_priv); 6919 6920 gen7_setup_fixed_func_scheduler(dev_priv); 6921 6922 if (0) { /* causes HiZ corruption on ivb:gt1 */ 6923 /* enable HiZ Raw Stall Optimization */ 6924 I915_WRITE(CACHE_MODE_0_GEN7, 6925 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE)); 6926 } 6927 6928 /* WaDisable4x2SubspanOptimization:ivb */ 6929 I915_WRITE(CACHE_MODE_1, 6930 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); 6931 6932 /* 6933 * BSpec recommends 8x4 when MSAA is used, 6934 * however in practice 16x4 seems fastest. 6935 * 6936 * Note that PS/WM thread counts depend on the WIZ hashing 6937 * disable bit, which we don't touch here, but it's good 6938 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 6939 */ 6940 I915_WRITE(GEN7_GT_MODE, 6941 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); 6942 6943 snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); 6944 snpcr &= ~GEN6_MBC_SNPCR_MASK; 6945 snpcr |= GEN6_MBC_SNPCR_MED; 6946 I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr); 6947 6948 if (!HAS_PCH_NOP(dev_priv)) 6949 cpt_init_clock_gating(dev_priv); 6950 6951 gen6_check_mch_setup(dev_priv); 6952 } 6953 6954 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv) 6955 { 6956 /* WaDisableEarlyCull:vlv */ 6957 I915_WRITE(_3D_CHICKEN3, 6958 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL)); 6959 6960 /* WaDisableBackToBackFlipFix:vlv */ 6961 I915_WRITE(IVB_CHICKEN3, 6962 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE | 6963 CHICKEN3_DGMG_DONE_FIX_DISABLE); 6964 6965 /* WaPsdDispatchEnable:vlv */ 6966 /* WaDisablePSDDualDispatchEnable:vlv */ 6967 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, 6968 _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP | 6969 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); 6970 6971 /* WaDisable_RenderCache_OperationalFlush:vlv */ 6972 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 6973 6974 /* WaForceL3Serialization:vlv */ 6975 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) & 6976 ~L3SQ_URB_READ_CAM_MATCH_DISABLE); 6977 6978 /* WaDisableDopClockGating:vlv */ 6979 I915_WRITE(GEN7_ROW_CHICKEN2, 6980 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); 6981 6982 /* This is required by WaCatErrorRejectionIssue:vlv */ 6983 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, 6984 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | 6985 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); 6986 6987 gen7_setup_fixed_func_scheduler(dev_priv); 6988 6989 /* 6990 * According to the spec, bit 13 (RCZUNIT) must be set on IVB. 6991 * This implements the WaDisableRCZUnitClockGating:vlv workaround. 6992 */ 6993 I915_WRITE(GEN6_UCGCTL2, 6994 GEN6_RCZUNIT_CLOCK_GATE_DISABLE); 6995 6996 /* WaDisableL3Bank2xClockGate:vlv 6997 * Disabling L3 clock gating- MMIO 940c[25] = 1 6998 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */ 6999 I915_WRITE(GEN7_UCGCTL4, 7000 I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE); 7001 7002 /* 7003 * BSpec says this must be set, even though 7004 * WaDisable4x2SubspanOptimization isn't listed for VLV. 7005 */ 7006 I915_WRITE(CACHE_MODE_1, 7007 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); 7008 7009 /* 7010 * BSpec recommends 8x4 when MSAA is used, 7011 * however in practice 16x4 seems fastest. 7012 * 7013 * Note that PS/WM thread counts depend on the WIZ hashing 7014 * disable bit, which we don't touch here, but it's good 7015 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 7016 */ 7017 I915_WRITE(GEN7_GT_MODE, 7018 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); 7019 7020 /* 7021 * WaIncreaseL3CreditsForVLVB0:vlv 7022 * This is the hardware default actually. 7023 */ 7024 I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE); 7025 7026 /* 7027 * WaDisableVLVClockGating_VBIIssue:vlv 7028 * Disable clock gating on th GCFG unit to prevent a delay 7029 * in the reporting of vblank events. 7030 */ 7031 I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS); 7032 } 7033 7034 static void chv_init_clock_gating(struct drm_i915_private *dev_priv) 7035 { 7036 /* WaVSRefCountFullforceMissDisable:chv */ 7037 /* WaDSRefCountFullforceMissDisable:chv */ 7038 I915_WRITE(GEN7_FF_THREAD_MODE, 7039 I915_READ(GEN7_FF_THREAD_MODE) & 7040 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME)); 7041 7042 /* WaDisableSemaphoreAndSyncFlipWait:chv */ 7043 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL, 7044 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE)); 7045 7046 /* WaDisableCSUnitClockGating:chv */ 7047 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) | 7048 GEN6_CSUNIT_CLOCK_GATE_DISABLE); 7049 7050 /* WaDisableSDEUnitClockGating:chv */ 7051 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | 7052 GEN8_SDEUNIT_CLOCK_GATE_DISABLE); 7053 7054 /* 7055 * WaProgramL3SqcReg1Default:chv 7056 * See gfxspecs/Related Documents/Performance Guide/ 7057 * LSQC Setting Recommendations. 7058 */ 7059 gen8_set_l3sqc_credits(dev_priv, 38, 2); 7060 } 7061 7062 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv) 7063 { 7064 u32 dspclk_gate; 7065 7066 I915_WRITE(RENCLK_GATE_D1, 0); 7067 I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE | 7068 GS_UNIT_CLOCK_GATE_DISABLE | 7069 CL_UNIT_CLOCK_GATE_DISABLE); 7070 I915_WRITE(RAMCLK_GATE_D, 0); 7071 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE | 7072 OVRUNIT_CLOCK_GATE_DISABLE | 7073 OVCUNIT_CLOCK_GATE_DISABLE; 7074 if (IS_GM45(dev_priv)) 7075 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE; 7076 I915_WRITE(DSPCLK_GATE_D, dspclk_gate); 7077 7078 /* WaDisableRenderCachePipelinedFlush */ 7079 I915_WRITE(CACHE_MODE_0, 7080 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE)); 7081 7082 /* WaDisable_RenderCache_OperationalFlush:g4x */ 7083 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 7084 7085 g4x_disable_trickle_feed(dev_priv); 7086 } 7087 7088 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv) 7089 { 7090 struct intel_uncore *uncore = &dev_priv->uncore; 7091 7092 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE); 7093 intel_uncore_write(uncore, RENCLK_GATE_D2, 0); 7094 intel_uncore_write(uncore, DSPCLK_GATE_D, 0); 7095 intel_uncore_write(uncore, RAMCLK_GATE_D, 0); 7096 intel_uncore_write16(uncore, DEUC, 0); 7097 intel_uncore_write(uncore, 7098 MI_ARB_STATE, 7099 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); 7100 7101 /* WaDisable_RenderCache_OperationalFlush:gen4 */ 7102 intel_uncore_write(uncore, 7103 CACHE_MODE_0, 7104 _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 7105 } 7106 7107 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv) 7108 { 7109 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE | 7110 I965_RCC_CLOCK_GATE_DISABLE | 7111 I965_RCPB_CLOCK_GATE_DISABLE | 7112 I965_ISC_CLOCK_GATE_DISABLE | 7113 I965_FBC_CLOCK_GATE_DISABLE); 7114 I915_WRITE(RENCLK_GATE_D2, 0); 7115 I915_WRITE(MI_ARB_STATE, 7116 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); 7117 7118 /* WaDisable_RenderCache_OperationalFlush:gen4 */ 7119 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 7120 } 7121 7122 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv) 7123 { 7124 u32 dstate = I915_READ(D_STATE); 7125 7126 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING | 7127 DSTATE_DOT_CLOCK_GATING; 7128 I915_WRITE(D_STATE, dstate); 7129 7130 if (IS_PINEVIEW(dev_priv)) 7131 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY)); 7132 7133 /* IIR "flip pending" means done if this bit is set */ 7134 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE)); 7135 7136 /* interrupts should cause a wake up from C3 */ 7137 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN)); 7138 7139 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 7140 I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); 7141 7142 I915_WRITE(MI_ARB_STATE, 7143 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); 7144 } 7145 7146 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv) 7147 { 7148 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE); 7149 7150 /* interrupts should cause a wake up from C3 */ 7151 I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) | 7152 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE)); 7153 7154 I915_WRITE(MEM_MODE, 7155 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE)); 7156 } 7157 7158 static void i830_init_clock_gating(struct drm_i915_private *dev_priv) 7159 { 7160 I915_WRITE(MEM_MODE, 7161 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) | 7162 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE)); 7163 } 7164 7165 void intel_init_clock_gating(struct drm_i915_private *dev_priv) 7166 { 7167 dev_priv->display.init_clock_gating(dev_priv); 7168 } 7169 7170 void intel_suspend_hw(struct drm_i915_private *dev_priv) 7171 { 7172 if (HAS_PCH_LPT(dev_priv)) 7173 lpt_suspend_hw(dev_priv); 7174 } 7175 7176 static void nop_init_clock_gating(struct drm_i915_private *dev_priv) 7177 { 7178 drm_dbg_kms(&dev_priv->drm, 7179 "No clock gating settings or workarounds applied.\n"); 7180 } 7181 7182 /** 7183 * intel_init_clock_gating_hooks - setup the clock gating hooks 7184 * @dev_priv: device private 7185 * 7186 * Setup the hooks that configure which clocks of a given platform can be 7187 * gated and also apply various GT and display specific workarounds for these 7188 * platforms. Note that some GT specific workarounds are applied separately 7189 * when GPU contexts or batchbuffers start their execution. 7190 */ 7191 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv) 7192 { 7193 if (IS_GEN(dev_priv, 12)) 7194 dev_priv->display.init_clock_gating = tgl_init_clock_gating; 7195 else if (IS_GEN(dev_priv, 11)) 7196 dev_priv->display.init_clock_gating = icl_init_clock_gating; 7197 else if (IS_CANNONLAKE(dev_priv)) 7198 dev_priv->display.init_clock_gating = cnl_init_clock_gating; 7199 else if (IS_COFFEELAKE(dev_priv)) 7200 dev_priv->display.init_clock_gating = cfl_init_clock_gating; 7201 else if (IS_SKYLAKE(dev_priv)) 7202 dev_priv->display.init_clock_gating = skl_init_clock_gating; 7203 else if (IS_KABYLAKE(dev_priv)) 7204 dev_priv->display.init_clock_gating = kbl_init_clock_gating; 7205 else if (IS_BROXTON(dev_priv)) 7206 dev_priv->display.init_clock_gating = bxt_init_clock_gating; 7207 else if (IS_GEMINILAKE(dev_priv)) 7208 dev_priv->display.init_clock_gating = glk_init_clock_gating; 7209 else if (IS_BROADWELL(dev_priv)) 7210 dev_priv->display.init_clock_gating = bdw_init_clock_gating; 7211 else if (IS_CHERRYVIEW(dev_priv)) 7212 dev_priv->display.init_clock_gating = chv_init_clock_gating; 7213 else if (IS_HASWELL(dev_priv)) 7214 dev_priv->display.init_clock_gating = hsw_init_clock_gating; 7215 else if (IS_IVYBRIDGE(dev_priv)) 7216 dev_priv->display.init_clock_gating = ivb_init_clock_gating; 7217 else if (IS_VALLEYVIEW(dev_priv)) 7218 dev_priv->display.init_clock_gating = vlv_init_clock_gating; 7219 else if (IS_GEN(dev_priv, 6)) 7220 dev_priv->display.init_clock_gating = gen6_init_clock_gating; 7221 else if (IS_GEN(dev_priv, 5)) 7222 dev_priv->display.init_clock_gating = ilk_init_clock_gating; 7223 else if (IS_G4X(dev_priv)) 7224 dev_priv->display.init_clock_gating = g4x_init_clock_gating; 7225 else if (IS_I965GM(dev_priv)) 7226 dev_priv->display.init_clock_gating = i965gm_init_clock_gating; 7227 else if (IS_I965G(dev_priv)) 7228 dev_priv->display.init_clock_gating = i965g_init_clock_gating; 7229 else if (IS_GEN(dev_priv, 3)) 7230 dev_priv->display.init_clock_gating = gen3_init_clock_gating; 7231 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv)) 7232 dev_priv->display.init_clock_gating = i85x_init_clock_gating; 7233 else if (IS_GEN(dev_priv, 2)) 7234 dev_priv->display.init_clock_gating = i830_init_clock_gating; 7235 else { 7236 MISSING_CASE(INTEL_DEVID(dev_priv)); 7237 dev_priv->display.init_clock_gating = nop_init_clock_gating; 7238 } 7239 } 7240 7241 /* Set up chip specific power management-related functions */ 7242 void intel_init_pm(struct drm_i915_private *dev_priv) 7243 { 7244 /* For cxsr */ 7245 if (IS_PINEVIEW(dev_priv)) 7246 pnv_get_mem_freq(dev_priv); 7247 else if (IS_GEN(dev_priv, 5)) 7248 ilk_get_mem_freq(dev_priv); 7249 7250 if (intel_has_sagv(dev_priv)) 7251 skl_setup_sagv_block_time(dev_priv); 7252 7253 /* For FIFO watermark updates */ 7254 if (INTEL_GEN(dev_priv) >= 9) { 7255 skl_setup_wm_latency(dev_priv); 7256 dev_priv->display.initial_watermarks = skl_initial_wm; 7257 dev_priv->display.atomic_update_watermarks = skl_atomic_update_crtc_wm; 7258 dev_priv->display.compute_global_watermarks = skl_compute_wm; 7259 } else if (HAS_PCH_SPLIT(dev_priv)) { 7260 ilk_setup_wm_latency(dev_priv); 7261 7262 if ((IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[1] && 7263 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) || 7264 (!IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[0] && 7265 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) { 7266 dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm; 7267 dev_priv->display.compute_intermediate_wm = 7268 ilk_compute_intermediate_wm; 7269 dev_priv->display.initial_watermarks = 7270 ilk_initial_watermarks; 7271 dev_priv->display.optimize_watermarks = 7272 ilk_optimize_watermarks; 7273 } else { 7274 drm_dbg_kms(&dev_priv->drm, 7275 "Failed to read display plane latency. " 7276 "Disable CxSR\n"); 7277 } 7278 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { 7279 vlv_setup_wm_latency(dev_priv); 7280 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm; 7281 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm; 7282 dev_priv->display.initial_watermarks = vlv_initial_watermarks; 7283 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks; 7284 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo; 7285 } else if (IS_G4X(dev_priv)) { 7286 g4x_setup_wm_latency(dev_priv); 7287 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm; 7288 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm; 7289 dev_priv->display.initial_watermarks = g4x_initial_watermarks; 7290 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks; 7291 } else if (IS_PINEVIEW(dev_priv)) { 7292 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv), 7293 dev_priv->is_ddr3, 7294 dev_priv->fsb_freq, 7295 dev_priv->mem_freq)) { 7296 drm_info(&dev_priv->drm, 7297 "failed to find known CxSR latency " 7298 "(found ddr%s fsb freq %d, mem freq %d), " 7299 "disabling CxSR\n", 7300 (dev_priv->is_ddr3 == 1) ? "3" : "2", 7301 dev_priv->fsb_freq, dev_priv->mem_freq); 7302 /* Disable CxSR and never update its watermark again */ 7303 intel_set_memory_cxsr(dev_priv, false); 7304 dev_priv->display.update_wm = NULL; 7305 } else 7306 dev_priv->display.update_wm = pnv_update_wm; 7307 } else if (IS_GEN(dev_priv, 4)) { 7308 dev_priv->display.update_wm = i965_update_wm; 7309 } else if (IS_GEN(dev_priv, 3)) { 7310 dev_priv->display.update_wm = i9xx_update_wm; 7311 dev_priv->display.get_fifo_size = i9xx_get_fifo_size; 7312 } else if (IS_GEN(dev_priv, 2)) { 7313 if (INTEL_NUM_PIPES(dev_priv) == 1) { 7314 dev_priv->display.update_wm = i845_update_wm; 7315 dev_priv->display.get_fifo_size = i845_get_fifo_size; 7316 } else { 7317 dev_priv->display.update_wm = i9xx_update_wm; 7318 dev_priv->display.get_fifo_size = i830_get_fifo_size; 7319 } 7320 } else { 7321 drm_err(&dev_priv->drm, 7322 "unexpected fall-through in %s\n", __func__); 7323 } 7324 } 7325 7326 void intel_pm_setup(struct drm_i915_private *dev_priv) 7327 { 7328 dev_priv->runtime_pm.suspended = false; 7329 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0); 7330 } 7331
Indexes created Sat Oct 11 19:10:01 GMT 2025