Home | History | Annotate | Line # | Download | only in radeon
radeon_rv6xx_dpm.c revision 1.1
      1 /*	$NetBSD: radeon_rv6xx_dpm.c,v 1.1 2018/08/27 14:38:20 riastradh Exp $	*/
      2 
      3 /*
      4  * Copyright 2011 Advanced Micro Devices, Inc.
      5  *
      6  * Permission is hereby granted, free of charge, to any person obtaining a
      7  * copy of this software and associated documentation files (the "Software"),
      8  * to deal in the Software without restriction, including without limitation
      9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
     10  * and/or sell copies of the Software, and to permit persons to whom the
     11  * Software is furnished to do so, subject to the following conditions:
     12  *
     13  * The above copyright notice and this permission notice shall be included in
     14  * all copies or substantial portions of the Software.
     15  *
     16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
     19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
     20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
     21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
     22  * OTHER DEALINGS IN THE SOFTWARE.
     23  *
     24  * Authors: Alex Deucher
     25  */
     26 
     27 #include <sys/cdefs.h>
     28 __KERNEL_RCSID(0, "$NetBSD: radeon_rv6xx_dpm.c,v 1.1 2018/08/27 14:38:20 riastradh Exp $");
     29 
     30 #include "drmP.h"
     31 #include "radeon.h"
     32 #include "radeon_asic.h"
     33 #include "rv6xxd.h"
     34 #include "r600_dpm.h"
     35 #include "rv6xx_dpm.h"
     36 #include "atom.h"
     37 #include <linux/seq_file.h>
     38 
     39 static u32 rv6xx_scale_count_given_unit(struct radeon_device *rdev,
     40 					u32 unscaled_count, u32 unit);
     41 
     42 static struct rv6xx_ps *rv6xx_get_ps(struct radeon_ps *rps)
     43 {
     44 	struct rv6xx_ps *ps = rps->ps_priv;
     45 
     46 	return ps;
     47 }
     48 
     49 static struct rv6xx_power_info *rv6xx_get_pi(struct radeon_device *rdev)
     50 {
     51 	struct rv6xx_power_info *pi = rdev->pm.dpm.priv;
     52 
     53 	return pi;
     54 }
     55 
     56 static void rv6xx_force_pcie_gen1(struct radeon_device *rdev)
     57 {
     58 	u32 tmp;
     59 	int i;
     60 
     61 	tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
     62 	tmp &= LC_GEN2_EN;
     63 	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
     64 
     65 	tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
     66 	tmp |= LC_INITIATE_LINK_SPEED_CHANGE;
     67 	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
     68 
     69 	for (i = 0; i < rdev->usec_timeout; i++) {
     70 		if (!(RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL) & LC_CURRENT_DATA_RATE))
     71 			break;
     72 		udelay(1);
     73 	}
     74 
     75 	tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
     76 	tmp &= ~LC_INITIATE_LINK_SPEED_CHANGE;
     77 	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
     78 }
     79 
     80 static void rv6xx_enable_pcie_gen2_support(struct radeon_device *rdev)
     81 {
     82 	u32 tmp;
     83 
     84 	tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
     85 
     86 	if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
     87 	    (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
     88 		tmp |= LC_GEN2_EN;
     89 		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
     90 	}
     91 }
     92 
     93 static void rv6xx_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
     94 					       bool enable)
     95 {
     96 	u32 tmp;
     97 
     98 	tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL) & ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
     99 	if (enable)
    100 		tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
    101 	else
    102 		tmp |= LC_HW_VOLTAGE_IF_CONTROL(0);
    103 	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
    104 }
    105 
    106 static void rv6xx_enable_l0s(struct radeon_device *rdev)
    107 {
    108 	u32 tmp;
    109 
    110 	tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L0S_INACTIVITY_MASK;
    111 	tmp |= LC_L0S_INACTIVITY(3);
    112 	WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
    113 }
    114 
    115 static void rv6xx_enable_l1(struct radeon_device *rdev)
    116 {
    117 	u32 tmp;
    118 
    119 	tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL);
    120 	tmp &= ~LC_L1_INACTIVITY_MASK;
    121 	tmp |= LC_L1_INACTIVITY(4);
    122 	tmp &= ~LC_PMI_TO_L1_DIS;
    123 	tmp &= ~LC_ASPM_TO_L1_DIS;
    124 	WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
    125 }
    126 
    127 static void rv6xx_enable_pll_sleep_in_l1(struct radeon_device *rdev)
    128 {
    129 	u32 tmp;
    130 
    131 	tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L1_INACTIVITY_MASK;
    132 	tmp |= LC_L1_INACTIVITY(8);
    133 	WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
    134 
    135 	/* NOTE, this is a PCIE indirect reg, not PCIE PORT */
    136 	tmp = RREG32_PCIE(PCIE_P_CNTL);
    137 	tmp |= P_PLL_PWRDN_IN_L1L23;
    138 	tmp &= ~P_PLL_BUF_PDNB;
    139 	tmp &= ~P_PLL_PDNB;
    140 	tmp |= P_ALLOW_PRX_FRONTEND_SHUTOFF;
    141 	WREG32_PCIE(PCIE_P_CNTL, tmp);
    142 }
    143 
    144 static int rv6xx_convert_clock_to_stepping(struct radeon_device *rdev,
    145 					   u32 clock, struct rv6xx_sclk_stepping *step)
    146 {
    147 	int ret;
    148 	struct atom_clock_dividers dividers;
    149 
    150 	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
    151 					     clock, false, &dividers);
    152 	if (ret)
    153 		return ret;
    154 
    155 	if (dividers.enable_post_div)
    156 		step->post_divider = 2 + (dividers.post_div & 0xF) + (dividers.post_div >> 4);
    157 	else
    158 		step->post_divider = 1;
    159 
    160 	step->vco_frequency = clock * step->post_divider;
    161 
    162 	return 0;
    163 }
    164 
    165 static void rv6xx_output_stepping(struct radeon_device *rdev,
    166 				  u32 step_index, struct rv6xx_sclk_stepping *step)
    167 {
    168 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    169 	u32 ref_clk = rdev->clock.spll.reference_freq;
    170 	u32 fb_divider;
    171 	u32 spll_step_count = rv6xx_scale_count_given_unit(rdev,
    172 							   R600_SPLLSTEPTIME_DFLT *
    173 							   pi->spll_ref_div,
    174 							   R600_SPLLSTEPUNIT_DFLT);
    175 
    176 	r600_engine_clock_entry_enable(rdev, step_index, true);
    177 	r600_engine_clock_entry_enable_pulse_skipping(rdev, step_index, false);
    178 
    179 	if (step->post_divider == 1)
    180 		r600_engine_clock_entry_enable_post_divider(rdev, step_index, false);
    181 	else {
    182 		u32 lo_len = (step->post_divider - 2) / 2;
    183 		u32 hi_len = step->post_divider - 2 - lo_len;
    184 
    185 		r600_engine_clock_entry_enable_post_divider(rdev, step_index, true);
    186 		r600_engine_clock_entry_set_post_divider(rdev, step_index, (hi_len << 4) | lo_len);
    187 	}
    188 
    189 	fb_divider = ((step->vco_frequency * pi->spll_ref_div) / ref_clk) >>
    190 		pi->fb_div_scale;
    191 
    192 	r600_engine_clock_entry_set_reference_divider(rdev, step_index,
    193 						      pi->spll_ref_div - 1);
    194 	r600_engine_clock_entry_set_feedback_divider(rdev, step_index, fb_divider);
    195 	r600_engine_clock_entry_set_step_time(rdev, step_index, spll_step_count);
    196 
    197 }
    198 
    199 static struct rv6xx_sclk_stepping rv6xx_next_vco_step(struct radeon_device *rdev,
    200 						      struct rv6xx_sclk_stepping *cur,
    201 						      bool increasing_vco, u32 step_size)
    202 {
    203 	struct rv6xx_sclk_stepping next;
    204 
    205 	next.post_divider = cur->post_divider;
    206 
    207 	if (increasing_vco)
    208 		next.vco_frequency = (cur->vco_frequency * (100 + step_size)) / 100;
    209 	else
    210 		next.vco_frequency = (cur->vco_frequency * 100 + 99 + step_size) / (100 + step_size);
    211 
    212 	return next;
    213 }
    214 
    215 static bool rv6xx_can_step_post_div(struct radeon_device *rdev,
    216 				    struct rv6xx_sclk_stepping *cur,
    217                                     struct rv6xx_sclk_stepping *target)
    218 {
    219 	return (cur->post_divider > target->post_divider) &&
    220 		((cur->vco_frequency * target->post_divider) <=
    221 		 (target->vco_frequency * (cur->post_divider - 1)));
    222 }
    223 
    224 static struct rv6xx_sclk_stepping rv6xx_next_post_div_step(struct radeon_device *rdev,
    225 							   struct rv6xx_sclk_stepping *cur,
    226 							   struct rv6xx_sclk_stepping *target)
    227 {
    228 	struct rv6xx_sclk_stepping next = *cur;
    229 
    230 	while (rv6xx_can_step_post_div(rdev, &next, target))
    231 		next.post_divider--;
    232 
    233 	return next;
    234 }
    235 
    236 static bool rv6xx_reached_stepping_target(struct radeon_device *rdev,
    237 					  struct rv6xx_sclk_stepping *cur,
    238 					  struct rv6xx_sclk_stepping *target,
    239 					  bool increasing_vco)
    240 {
    241 	return (increasing_vco && (cur->vco_frequency >= target->vco_frequency)) ||
    242 		(!increasing_vco && (cur->vco_frequency <= target->vco_frequency));
    243 }
    244 
    245 static void rv6xx_generate_steps(struct radeon_device *rdev,
    246 				 u32 low, u32 high,
    247                                  u32 start_index, u8 *end_index)
    248 {
    249 	struct rv6xx_sclk_stepping cur;
    250 	struct rv6xx_sclk_stepping target;
    251 	bool increasing_vco;
    252 	u32 step_index = start_index;
    253 
    254 	rv6xx_convert_clock_to_stepping(rdev, low, &cur);
    255 	rv6xx_convert_clock_to_stepping(rdev, high, &target);
    256 
    257 	rv6xx_output_stepping(rdev, step_index++, &cur);
    258 
    259 	increasing_vco = (target.vco_frequency >= cur.vco_frequency);
    260 
    261 	if (target.post_divider > cur.post_divider)
    262 		cur.post_divider = target.post_divider;
    263 
    264 	while (1) {
    265 		struct rv6xx_sclk_stepping next;
    266 
    267 		if (rv6xx_can_step_post_div(rdev, &cur, &target))
    268 			next = rv6xx_next_post_div_step(rdev, &cur, &target);
    269 		else
    270 			next = rv6xx_next_vco_step(rdev, &cur, increasing_vco, R600_VCOSTEPPCT_DFLT);
    271 
    272 		if (rv6xx_reached_stepping_target(rdev, &next, &target, increasing_vco)) {
    273 			struct rv6xx_sclk_stepping tiny =
    274 				rv6xx_next_vco_step(rdev, &target, !increasing_vco, R600_ENDINGVCOSTEPPCT_DFLT);
    275 			tiny.post_divider = next.post_divider;
    276 
    277 			if (!rv6xx_reached_stepping_target(rdev, &tiny, &cur, !increasing_vco))
    278 				rv6xx_output_stepping(rdev, step_index++, &tiny);
    279 
    280 			if ((next.post_divider != target.post_divider) &&
    281 			    (next.vco_frequency != target.vco_frequency)) {
    282 				struct rv6xx_sclk_stepping final_vco;
    283 
    284 				final_vco.vco_frequency = target.vco_frequency;
    285 				final_vco.post_divider = next.post_divider;
    286 
    287 				rv6xx_output_stepping(rdev, step_index++, &final_vco);
    288 			}
    289 
    290 			rv6xx_output_stepping(rdev, step_index++, &target);
    291 			break;
    292 		} else
    293 			rv6xx_output_stepping(rdev, step_index++, &next);
    294 
    295 		cur = next;
    296 	}
    297 
    298 	*end_index = (u8)step_index - 1;
    299 
    300 }
    301 
    302 static void rv6xx_generate_single_step(struct radeon_device *rdev,
    303 				       u32 clock, u32 index)
    304 {
    305 	struct rv6xx_sclk_stepping step;
    306 
    307 	rv6xx_convert_clock_to_stepping(rdev, clock, &step);
    308 	rv6xx_output_stepping(rdev, index, &step);
    309 }
    310 
    311 static void rv6xx_invalidate_intermediate_steps_range(struct radeon_device *rdev,
    312 						      u32 start_index, u32 end_index)
    313 {
    314 	u32 step_index;
    315 
    316 	for (step_index = start_index + 1; step_index < end_index; step_index++)
    317 		r600_engine_clock_entry_enable(rdev, step_index, false);
    318 }
    319 
    320 static void rv6xx_set_engine_spread_spectrum_clk_s(struct radeon_device *rdev,
    321 						   u32 index, u32 clk_s)
    322 {
    323 	WREG32_P(CG_SPLL_SPREAD_SPECTRUM_LOW + (index * 4),
    324 		 CLKS(clk_s), ~CLKS_MASK);
    325 }
    326 
    327 static void rv6xx_set_engine_spread_spectrum_clk_v(struct radeon_device *rdev,
    328 						   u32 index, u32 clk_v)
    329 {
    330 	WREG32_P(CG_SPLL_SPREAD_SPECTRUM_LOW + (index * 4),
    331 		 CLKV(clk_v), ~CLKV_MASK);
    332 }
    333 
    334 static void rv6xx_enable_engine_spread_spectrum(struct radeon_device *rdev,
    335 						u32 index, bool enable)
    336 {
    337 	if (enable)
    338 		WREG32_P(CG_SPLL_SPREAD_SPECTRUM_LOW + (index * 4),
    339 			 SSEN, ~SSEN);
    340 	else
    341 		WREG32_P(CG_SPLL_SPREAD_SPECTRUM_LOW + (index * 4),
    342 			 0, ~SSEN);
    343 }
    344 
    345 static void rv6xx_set_memory_spread_spectrum_clk_s(struct radeon_device *rdev,
    346 						   u32 clk_s)
    347 {
    348 	WREG32_P(CG_MPLL_SPREAD_SPECTRUM, CLKS(clk_s), ~CLKS_MASK);
    349 }
    350 
    351 static void rv6xx_set_memory_spread_spectrum_clk_v(struct radeon_device *rdev,
    352 						   u32 clk_v)
    353 {
    354 	WREG32_P(CG_MPLL_SPREAD_SPECTRUM, CLKV(clk_v), ~CLKV_MASK);
    355 }
    356 
    357 static void rv6xx_enable_memory_spread_spectrum(struct radeon_device *rdev,
    358 						bool enable)
    359 {
    360 	if (enable)
    361 		WREG32_P(CG_MPLL_SPREAD_SPECTRUM, SSEN, ~SSEN);
    362 	else
    363 		WREG32_P(CG_MPLL_SPREAD_SPECTRUM, 0, ~SSEN);
    364 }
    365 
    366 static void rv6xx_enable_dynamic_spread_spectrum(struct radeon_device *rdev,
    367 						 bool enable)
    368 {
    369 	if (enable)
    370 		WREG32_P(GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, ~DYN_SPREAD_SPECTRUM_EN);
    371 	else
    372 		WREG32_P(GENERAL_PWRMGT, 0, ~DYN_SPREAD_SPECTRUM_EN);
    373 }
    374 
    375 static void rv6xx_memory_clock_entry_enable_post_divider(struct radeon_device *rdev,
    376 							 u32 index, bool enable)
    377 {
    378 	if (enable)
    379 		WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4),
    380 			 LEVEL0_MPLL_DIV_EN, ~LEVEL0_MPLL_DIV_EN);
    381 	else
    382 		WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4), 0, ~LEVEL0_MPLL_DIV_EN);
    383 }
    384 
    385 static void rv6xx_memory_clock_entry_set_post_divider(struct radeon_device *rdev,
    386 						      u32 index, u32 divider)
    387 {
    388 	WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4),
    389 		 LEVEL0_MPLL_POST_DIV(divider), ~LEVEL0_MPLL_POST_DIV_MASK);
    390 }
    391 
    392 static void rv6xx_memory_clock_entry_set_feedback_divider(struct radeon_device *rdev,
    393 							  u32 index, u32 divider)
    394 {
    395 	WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4), LEVEL0_MPLL_FB_DIV(divider),
    396 		 ~LEVEL0_MPLL_FB_DIV_MASK);
    397 }
    398 
    399 static void rv6xx_memory_clock_entry_set_reference_divider(struct radeon_device *rdev,
    400 							   u32 index, u32 divider)
    401 {
    402 	WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4),
    403 		 LEVEL0_MPLL_REF_DIV(divider), ~LEVEL0_MPLL_REF_DIV_MASK);
    404 }
    405 
    406 static void rv6xx_vid_response_set_brt(struct radeon_device *rdev, u32 rt)
    407 {
    408 	WREG32_P(VID_RT, BRT(rt), ~BRT_MASK);
    409 }
    410 
    411 static void rv6xx_enable_engine_feedback_and_reference_sync(struct radeon_device *rdev)
    412 {
    413 	WREG32_P(SPLL_CNTL_MODE, SPLL_DIV_SYNC, ~SPLL_DIV_SYNC);
    414 }
    415 
    416 static u32 rv6xx_clocks_per_unit(u32 unit)
    417 {
    418 	u32 tmp = 1 << (2 * unit);
    419 
    420 	return tmp;
    421 }
    422 
    423 static u32 rv6xx_scale_count_given_unit(struct radeon_device *rdev,
    424 					u32 unscaled_count, u32 unit)
    425 {
    426 	u32 count_per_unit = rv6xx_clocks_per_unit(unit);
    427 
    428 	return (unscaled_count + count_per_unit - 1) / count_per_unit;
    429 }
    430 
    431 static u32 rv6xx_compute_count_for_delay(struct radeon_device *rdev,
    432 					 u32 delay_us, u32 unit)
    433 {
    434 	u32 ref_clk = rdev->clock.spll.reference_freq;
    435 
    436 	return rv6xx_scale_count_given_unit(rdev, delay_us * (ref_clk / 100), unit);
    437 }
    438 
    439 static void rv6xx_calculate_engine_speed_stepping_parameters(struct radeon_device *rdev,
    440 							     struct rv6xx_ps *state)
    441 {
    442 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    443 
    444 	pi->hw.sclks[R600_POWER_LEVEL_LOW] =
    445 		state->low.sclk;
    446 	pi->hw.sclks[R600_POWER_LEVEL_MEDIUM] =
    447 		state->medium.sclk;
    448 	pi->hw.sclks[R600_POWER_LEVEL_HIGH] =
    449 		state->high.sclk;
    450 
    451 	pi->hw.low_sclk_index = R600_POWER_LEVEL_LOW;
    452 	pi->hw.medium_sclk_index = R600_POWER_LEVEL_MEDIUM;
    453 	pi->hw.high_sclk_index = R600_POWER_LEVEL_HIGH;
    454 }
    455 
    456 static void rv6xx_calculate_memory_clock_stepping_parameters(struct radeon_device *rdev,
    457 							     struct rv6xx_ps *state)
    458 {
    459 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    460 
    461 	pi->hw.mclks[R600_POWER_LEVEL_CTXSW] =
    462 		state->high.mclk;
    463 	pi->hw.mclks[R600_POWER_LEVEL_HIGH] =
    464 		state->high.mclk;
    465 	pi->hw.mclks[R600_POWER_LEVEL_MEDIUM] =
    466 		state->medium.mclk;
    467 	pi->hw.mclks[R600_POWER_LEVEL_LOW] =
    468 		state->low.mclk;
    469 
    470 	pi->hw.high_mclk_index = R600_POWER_LEVEL_HIGH;
    471 
    472 	if (state->high.mclk == state->medium.mclk)
    473 		pi->hw.medium_mclk_index =
    474 			pi->hw.high_mclk_index;
    475 	else
    476 		pi->hw.medium_mclk_index = R600_POWER_LEVEL_MEDIUM;
    477 
    478 
    479 	if (state->medium.mclk == state->low.mclk)
    480 		pi->hw.low_mclk_index =
    481 			pi->hw.medium_mclk_index;
    482 	else
    483 		pi->hw.low_mclk_index = R600_POWER_LEVEL_LOW;
    484 }
    485 
    486 static void rv6xx_calculate_voltage_stepping_parameters(struct radeon_device *rdev,
    487 							struct rv6xx_ps *state)
    488 {
    489 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    490 
    491 	pi->hw.vddc[R600_POWER_LEVEL_CTXSW] = state->high.vddc;
    492 	pi->hw.vddc[R600_POWER_LEVEL_HIGH] = state->high.vddc;
    493 	pi->hw.vddc[R600_POWER_LEVEL_MEDIUM] = state->medium.vddc;
    494 	pi->hw.vddc[R600_POWER_LEVEL_LOW] = state->low.vddc;
    495 
    496 	pi->hw.backbias[R600_POWER_LEVEL_CTXSW] =
    497 		(state->high.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? true : false;
    498 	pi->hw.backbias[R600_POWER_LEVEL_HIGH] =
    499 		(state->high.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? true : false;
    500 	pi->hw.backbias[R600_POWER_LEVEL_MEDIUM] =
    501 		(state->medium.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? true : false;
    502 	pi->hw.backbias[R600_POWER_LEVEL_LOW] =
    503 		(state->low.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? true : false;
    504 
    505 	pi->hw.pcie_gen2[R600_POWER_LEVEL_HIGH] =
    506 		(state->high.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? true : false;
    507 	pi->hw.pcie_gen2[R600_POWER_LEVEL_MEDIUM] =
    508 		(state->medium.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? true : false;
    509 	pi->hw.pcie_gen2[R600_POWER_LEVEL_LOW] =
    510 		(state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? true : false;
    511 
    512 	pi->hw.high_vddc_index = R600_POWER_LEVEL_HIGH;
    513 
    514 	if ((state->high.vddc == state->medium.vddc) &&
    515 	    ((state->high.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ==
    516 	     (state->medium.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE)))
    517 		pi->hw.medium_vddc_index =
    518 			pi->hw.high_vddc_index;
    519 	else
    520 		pi->hw.medium_vddc_index = R600_POWER_LEVEL_MEDIUM;
    521 
    522 	if ((state->medium.vddc == state->low.vddc) &&
    523 	    ((state->medium.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ==
    524 	     (state->low.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE)))
    525 		pi->hw.low_vddc_index =
    526 			pi->hw.medium_vddc_index;
    527 	else
    528 		pi->hw.medium_vddc_index = R600_POWER_LEVEL_LOW;
    529 }
    530 
    531 static inline u32 rv6xx_calculate_vco_frequency(u32 ref_clock,
    532 						struct atom_clock_dividers *dividers,
    533 						u32 fb_divider_scale)
    534 {
    535 	return ref_clock * ((dividers->fb_div & ~1) << fb_divider_scale) /
    536 		(dividers->ref_div + 1);
    537 }
    538 
    539 static inline u32 rv6xx_calculate_spread_spectrum_clk_v(u32 vco_freq, u32 ref_freq,
    540 							u32 ss_rate, u32 ss_percent,
    541 							u32 fb_divider_scale)
    542 {
    543 	u32 fb_divider = vco_freq / ref_freq;
    544 
    545 	return (ss_percent * ss_rate * 4 * (fb_divider * fb_divider) /
    546 		(5375 * ((vco_freq * 10) / (4096 >> fb_divider_scale))));
    547 }
    548 
    549 static inline u32 rv6xx_calculate_spread_spectrum_clk_s(u32 ss_rate, u32 ref_freq)
    550 {
    551 	return (((ref_freq * 10) / (ss_rate * 2)) - 1) / 4;
    552 }
    553 
    554 static void rv6xx_program_engine_spread_spectrum(struct radeon_device *rdev,
    555 						 u32 clock, enum r600_power_level level)
    556 {
    557 	u32 ref_clk = rdev->clock.spll.reference_freq;
    558 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    559 	struct atom_clock_dividers dividers;
    560 	struct radeon_atom_ss ss;
    561 	u32 vco_freq, clk_v, clk_s;
    562 
    563 	rv6xx_enable_engine_spread_spectrum(rdev, level, false);
    564 
    565 	if (clock && pi->sclk_ss) {
    566 		if (radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, clock, false, &dividers) == 0) {
    567 			vco_freq = rv6xx_calculate_vco_frequency(ref_clk, &dividers,
    568 								 pi->fb_div_scale);
    569 
    570 			if (radeon_atombios_get_asic_ss_info(rdev, &ss,
    571 							     ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
    572 				clk_v = rv6xx_calculate_spread_spectrum_clk_v(vco_freq,
    573 									      (ref_clk / (dividers.ref_div + 1)),
    574 									      ss.rate,
    575 									      ss.percentage,
    576 									      pi->fb_div_scale);
    577 
    578 				clk_s = rv6xx_calculate_spread_spectrum_clk_s(ss.rate,
    579 									      (ref_clk / (dividers.ref_div + 1)));
    580 
    581 				rv6xx_set_engine_spread_spectrum_clk_v(rdev, level, clk_v);
    582 				rv6xx_set_engine_spread_spectrum_clk_s(rdev, level, clk_s);
    583 				rv6xx_enable_engine_spread_spectrum(rdev, level, true);
    584 			}
    585 		}
    586 	}
    587 }
    588 
    589 static void rv6xx_program_sclk_spread_spectrum_parameters_except_lowest_entry(struct radeon_device *rdev)
    590 {
    591 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    592 
    593 	rv6xx_program_engine_spread_spectrum(rdev,
    594 					     pi->hw.sclks[R600_POWER_LEVEL_HIGH],
    595 					     R600_POWER_LEVEL_HIGH);
    596 
    597 	rv6xx_program_engine_spread_spectrum(rdev,
    598 					     pi->hw.sclks[R600_POWER_LEVEL_MEDIUM],
    599 					     R600_POWER_LEVEL_MEDIUM);
    600 
    601 }
    602 
    603 static int rv6xx_program_mclk_stepping_entry(struct radeon_device *rdev,
    604 					     u32 entry, u32 clock)
    605 {
    606 	struct atom_clock_dividers dividers;
    607 
    608 	if (radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM, clock, false, &dividers))
    609 	    return -EINVAL;
    610 
    611 
    612 	rv6xx_memory_clock_entry_set_reference_divider(rdev, entry, dividers.ref_div);
    613 	rv6xx_memory_clock_entry_set_feedback_divider(rdev, entry, dividers.fb_div);
    614 	rv6xx_memory_clock_entry_set_post_divider(rdev, entry, dividers.post_div);
    615 
    616 	if (dividers.enable_post_div)
    617 		rv6xx_memory_clock_entry_enable_post_divider(rdev, entry, true);
    618 	else
    619 		rv6xx_memory_clock_entry_enable_post_divider(rdev, entry, false);
    620 
    621 	return 0;
    622 }
    623 
    624 static void rv6xx_program_mclk_stepping_parameters_except_lowest_entry(struct radeon_device *rdev)
    625 {
    626 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    627 	int i;
    628 
    629 	for (i = 1; i < R600_PM_NUMBER_OF_MCLKS; i++) {
    630 		if (pi->hw.mclks[i])
    631 			rv6xx_program_mclk_stepping_entry(rdev, i,
    632 							  pi->hw.mclks[i]);
    633 	}
    634 }
    635 
    636 static void rv6xx_find_memory_clock_with_highest_vco(struct radeon_device *rdev,
    637 						     u32 requested_memory_clock,
    638 						     u32 ref_clk,
    639 						     struct atom_clock_dividers *dividers,
    640 						     u32 *vco_freq)
    641 {
    642 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    643 	struct atom_clock_dividers req_dividers;
    644 	u32 vco_freq_temp;
    645 
    646 	if (radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
    647 					   requested_memory_clock, false, &req_dividers) == 0) {
    648 		vco_freq_temp = rv6xx_calculate_vco_frequency(ref_clk, &req_dividers,
    649 							      pi->fb_div_scale);
    650 
    651 		if (vco_freq_temp > *vco_freq) {
    652 			*dividers = req_dividers;
    653 			*vco_freq = vco_freq_temp;
    654 		}
    655 	}
    656 }
    657 
    658 static void rv6xx_program_mclk_spread_spectrum_parameters(struct radeon_device *rdev)
    659 {
    660 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    661 	u32 ref_clk = rdev->clock.mpll.reference_freq;
    662 	struct atom_clock_dividers dividers;
    663 	struct radeon_atom_ss ss;
    664 	u32 vco_freq = 0, clk_v, clk_s;
    665 
    666 	rv6xx_enable_memory_spread_spectrum(rdev, false);
    667 
    668 	if (pi->mclk_ss) {
    669 		rv6xx_find_memory_clock_with_highest_vco(rdev,
    670 							 pi->hw.mclks[pi->hw.high_mclk_index],
    671 							 ref_clk,
    672 							 &dividers,
    673 							 &vco_freq);
    674 
    675 		rv6xx_find_memory_clock_with_highest_vco(rdev,
    676 							 pi->hw.mclks[pi->hw.medium_mclk_index],
    677 							 ref_clk,
    678 							 &dividers,
    679 							 &vco_freq);
    680 
    681 		rv6xx_find_memory_clock_with_highest_vco(rdev,
    682 							 pi->hw.mclks[pi->hw.low_mclk_index],
    683 							 ref_clk,
    684 							 &dividers,
    685 							 &vco_freq);
    686 
    687 		if (vco_freq) {
    688 			if (radeon_atombios_get_asic_ss_info(rdev, &ss,
    689 							     ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
    690 				clk_v = rv6xx_calculate_spread_spectrum_clk_v(vco_freq,
    691 									     (ref_clk / (dividers.ref_div + 1)),
    692 									     ss.rate,
    693 									     ss.percentage,
    694 									     pi->fb_div_scale);
    695 
    696 				clk_s = rv6xx_calculate_spread_spectrum_clk_s(ss.rate,
    697 									     (ref_clk / (dividers.ref_div + 1)));
    698 
    699 				rv6xx_set_memory_spread_spectrum_clk_v(rdev, clk_v);
    700 				rv6xx_set_memory_spread_spectrum_clk_s(rdev, clk_s);
    701 				rv6xx_enable_memory_spread_spectrum(rdev, true);
    702 			}
    703 		}
    704 	}
    705 }
    706 
    707 static int rv6xx_program_voltage_stepping_entry(struct radeon_device *rdev,
    708 						u32 entry, u16 voltage)
    709 {
    710 	u32 mask, set_pins;
    711 	int ret;
    712 
    713 	ret = radeon_atom_get_voltage_gpio_settings(rdev, voltage,
    714 						    SET_VOLTAGE_TYPE_ASIC_VDDC,
    715 						    &set_pins, &mask);
    716 	if (ret)
    717 		return ret;
    718 
    719 	r600_voltage_control_program_voltages(rdev, entry, set_pins);
    720 
    721 	return 0;
    722 }
    723 
    724 static void rv6xx_program_voltage_stepping_parameters_except_lowest_entry(struct radeon_device *rdev)
    725 {
    726 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    727 	int i;
    728 
    729 	for (i = 1; i < R600_PM_NUMBER_OF_VOLTAGE_LEVELS; i++)
    730 		rv6xx_program_voltage_stepping_entry(rdev, i,
    731 						     pi->hw.vddc[i]);
    732 
    733 }
    734 
    735 static void rv6xx_program_backbias_stepping_parameters_except_lowest_entry(struct radeon_device *rdev)
    736 {
    737 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    738 
    739 	if (pi->hw.backbias[1])
    740 		WREG32_P(VID_UPPER_GPIO_CNTL, MEDIUM_BACKBIAS_VALUE, ~MEDIUM_BACKBIAS_VALUE);
    741 	else
    742 		WREG32_P(VID_UPPER_GPIO_CNTL, 0, ~MEDIUM_BACKBIAS_VALUE);
    743 
    744 	if (pi->hw.backbias[2])
    745 		WREG32_P(VID_UPPER_GPIO_CNTL, HIGH_BACKBIAS_VALUE, ~HIGH_BACKBIAS_VALUE);
    746 	else
    747 		WREG32_P(VID_UPPER_GPIO_CNTL, 0, ~HIGH_BACKBIAS_VALUE);
    748 }
    749 
    750 static void rv6xx_program_sclk_spread_spectrum_parameters_lowest_entry(struct radeon_device *rdev)
    751 {
    752 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    753 
    754 	rv6xx_program_engine_spread_spectrum(rdev,
    755 					     pi->hw.sclks[R600_POWER_LEVEL_LOW],
    756 					     R600_POWER_LEVEL_LOW);
    757 }
    758 
    759 static void rv6xx_program_mclk_stepping_parameters_lowest_entry(struct radeon_device *rdev)
    760 {
    761 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    762 
    763 	if (pi->hw.mclks[0])
    764 		rv6xx_program_mclk_stepping_entry(rdev, 0,
    765 						  pi->hw.mclks[0]);
    766 }
    767 
    768 static void rv6xx_program_voltage_stepping_parameters_lowest_entry(struct radeon_device *rdev)
    769 {
    770 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    771 
    772 	rv6xx_program_voltage_stepping_entry(rdev, 0,
    773 					     pi->hw.vddc[0]);
    774 
    775 }
    776 
    777 static void rv6xx_program_backbias_stepping_parameters_lowest_entry(struct radeon_device *rdev)
    778 {
    779 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    780 
    781 	if (pi->hw.backbias[0])
    782 		WREG32_P(VID_UPPER_GPIO_CNTL, LOW_BACKBIAS_VALUE, ~LOW_BACKBIAS_VALUE);
    783 	else
    784 		WREG32_P(VID_UPPER_GPIO_CNTL, 0, ~LOW_BACKBIAS_VALUE);
    785 }
    786 
    787 static u32 calculate_memory_refresh_rate(struct radeon_device *rdev,
    788 					 u32 engine_clock)
    789 {
    790 	u32 dram_rows, dram_refresh_rate;
    791 	u32 tmp;
    792 
    793 	tmp = (RREG32(RAMCFG) & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
    794 	dram_rows = 1 << (tmp + 10);
    795 	dram_refresh_rate = 1 << ((RREG32(MC_SEQ_RESERVE_M) & 0x3) + 3);
    796 
    797 	return ((engine_clock * 10) * dram_refresh_rate / dram_rows - 32) / 64;
    798 }
    799 
    800 static void rv6xx_program_memory_timing_parameters(struct radeon_device *rdev)
    801 {
    802 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    803 	u32 sqm_ratio;
    804 	u32 arb_refresh_rate;
    805 	u32 high_clock;
    806 
    807 	if (pi->hw.sclks[R600_POWER_LEVEL_HIGH] <
    808 	    (pi->hw.sclks[R600_POWER_LEVEL_LOW] * 0xFF / 0x40))
    809 		high_clock = pi->hw.sclks[R600_POWER_LEVEL_HIGH];
    810 	else
    811 		high_clock =
    812 			pi->hw.sclks[R600_POWER_LEVEL_LOW] * 0xFF / 0x40;
    813 
    814 	radeon_atom_set_engine_dram_timings(rdev, high_clock, 0);
    815 
    816 	sqm_ratio = (STATE0(64 * high_clock / pi->hw.sclks[R600_POWER_LEVEL_LOW]) |
    817 		     STATE1(64 * high_clock / pi->hw.sclks[R600_POWER_LEVEL_MEDIUM]) |
    818 		     STATE2(64 * high_clock / pi->hw.sclks[R600_POWER_LEVEL_HIGH]) |
    819 		     STATE3(64 * high_clock / pi->hw.sclks[R600_POWER_LEVEL_HIGH]));
    820 	WREG32(SQM_RATIO, sqm_ratio);
    821 
    822 	arb_refresh_rate =
    823 		(POWERMODE0(calculate_memory_refresh_rate(rdev,
    824 							  pi->hw.sclks[R600_POWER_LEVEL_LOW])) |
    825 		 POWERMODE1(calculate_memory_refresh_rate(rdev,
    826 							  pi->hw.sclks[R600_POWER_LEVEL_MEDIUM])) |
    827 		 POWERMODE2(calculate_memory_refresh_rate(rdev,
    828 							  pi->hw.sclks[R600_POWER_LEVEL_HIGH])) |
    829 		 POWERMODE3(calculate_memory_refresh_rate(rdev,
    830 							  pi->hw.sclks[R600_POWER_LEVEL_HIGH])));
    831 	WREG32(ARB_RFSH_RATE, arb_refresh_rate);
    832 }
    833 
    834 static void rv6xx_program_mpll_timing_parameters(struct radeon_device *rdev)
    835 {
    836 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    837 
    838 	r600_set_mpll_lock_time(rdev, R600_MPLLLOCKTIME_DFLT *
    839 				pi->mpll_ref_div);
    840 	r600_set_mpll_reset_time(rdev, R600_MPLLRESETTIME_DFLT);
    841 }
    842 
    843 static void rv6xx_program_bsp(struct radeon_device *rdev)
    844 {
    845 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    846 	u32 ref_clk = rdev->clock.spll.reference_freq;
    847 
    848 	r600_calculate_u_and_p(R600_ASI_DFLT,
    849 			       ref_clk, 16,
    850 			       &pi->bsp,
    851 			       &pi->bsu);
    852 
    853 	r600_set_bsp(rdev, pi->bsu, pi->bsp);
    854 }
    855 
    856 static void rv6xx_program_at(struct radeon_device *rdev)
    857 {
    858 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    859 
    860 	r600_set_at(rdev,
    861 		    (pi->hw.rp[0] * pi->bsp) / 200,
    862 		    (pi->hw.rp[1] * pi->bsp) / 200,
    863 		    (pi->hw.lp[2] * pi->bsp) / 200,
    864 		    (pi->hw.lp[1] * pi->bsp) / 200);
    865 }
    866 
    867 static void rv6xx_program_git(struct radeon_device *rdev)
    868 {
    869 	r600_set_git(rdev, R600_GICST_DFLT);
    870 }
    871 
    872 static void rv6xx_program_tp(struct radeon_device *rdev)
    873 {
    874 	int i;
    875 
    876 	for (i = 0; i < R600_PM_NUMBER_OF_TC; i++)
    877 		r600_set_tc(rdev, i, r600_utc[i], r600_dtc[i]);
    878 
    879 	r600_select_td(rdev, R600_TD_DFLT);
    880 }
    881 
    882 static void rv6xx_program_vc(struct radeon_device *rdev)
    883 {
    884 	r600_set_vrc(rdev, R600_VRC_DFLT);
    885 }
    886 
    887 static void rv6xx_clear_vc(struct radeon_device *rdev)
    888 {
    889 	r600_set_vrc(rdev, 0);
    890 }
    891 
    892 static void rv6xx_program_tpp(struct radeon_device *rdev)
    893 {
    894 	r600_set_tpu(rdev, R600_TPU_DFLT);
    895 	r600_set_tpc(rdev, R600_TPC_DFLT);
    896 }
    897 
    898 static void rv6xx_program_sstp(struct radeon_device *rdev)
    899 {
    900 	r600_set_sstu(rdev, R600_SSTU_DFLT);
    901 	r600_set_sst(rdev, R600_SST_DFLT);
    902 }
    903 
    904 static void rv6xx_program_fcp(struct radeon_device *rdev)
    905 {
    906 	r600_set_fctu(rdev, R600_FCTU_DFLT);
    907 	r600_set_fct(rdev, R600_FCT_DFLT);
    908 }
    909 
    910 static void rv6xx_program_vddc3d_parameters(struct radeon_device *rdev)
    911 {
    912 	r600_set_vddc3d_oorsu(rdev, R600_VDDC3DOORSU_DFLT);
    913 	r600_set_vddc3d_oorphc(rdev, R600_VDDC3DOORPHC_DFLT);
    914 	r600_set_vddc3d_oorsdc(rdev, R600_VDDC3DOORSDC_DFLT);
    915 	r600_set_ctxcgtt3d_rphc(rdev, R600_CTXCGTT3DRPHC_DFLT);
    916 	r600_set_ctxcgtt3d_rsdc(rdev, R600_CTXCGTT3DRSDC_DFLT);
    917 }
    918 
    919 static void rv6xx_program_voltage_timing_parameters(struct radeon_device *rdev)
    920 {
    921 	u32 rt;
    922 
    923 	r600_vid_rt_set_vru(rdev, R600_VRU_DFLT);
    924 
    925 	r600_vid_rt_set_vrt(rdev,
    926 			    rv6xx_compute_count_for_delay(rdev,
    927 							  rdev->pm.dpm.voltage_response_time,
    928 							  R600_VRU_DFLT));
    929 
    930 	rt = rv6xx_compute_count_for_delay(rdev,
    931 					   rdev->pm.dpm.backbias_response_time,
    932 					   R600_VRU_DFLT);
    933 
    934 	rv6xx_vid_response_set_brt(rdev, (rt + 0x1F) >> 5);
    935 }
    936 
    937 static void rv6xx_program_engine_speed_parameters(struct radeon_device *rdev)
    938 {
    939 	r600_vid_rt_set_ssu(rdev, R600_SPLLSTEPUNIT_DFLT);
    940 	rv6xx_enable_engine_feedback_and_reference_sync(rdev);
    941 }
    942 
    943 static u64 rv6xx_get_master_voltage_mask(struct radeon_device *rdev)
    944 {
    945 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
    946 	u64 master_mask = 0;
    947 	int i;
    948 
    949 	for (i = 0; i < R600_PM_NUMBER_OF_VOLTAGE_LEVELS; i++) {
    950 		u32 tmp_mask, tmp_set_pins;
    951 		int ret;
    952 
    953 		ret = radeon_atom_get_voltage_gpio_settings(rdev,
    954 							    pi->hw.vddc[i],
    955 							    SET_VOLTAGE_TYPE_ASIC_VDDC,
    956 							    &tmp_set_pins, &tmp_mask);
    957 
    958 		if (ret == 0)
    959 			master_mask |= tmp_mask;
    960 	}
    961 
    962 	return master_mask;
    963 }
    964 
    965 static void rv6xx_program_voltage_gpio_pins(struct radeon_device *rdev)
    966 {
    967 	r600_voltage_control_enable_pins(rdev,
    968 					 rv6xx_get_master_voltage_mask(rdev));
    969 }
    970 
    971 static void rv6xx_enable_static_voltage_control(struct radeon_device *rdev,
    972 						struct radeon_ps *new_ps,
    973 						bool enable)
    974 {
    975 	struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
    976 
    977 	if (enable)
    978 		radeon_atom_set_voltage(rdev,
    979 					new_state->low.vddc,
    980 					SET_VOLTAGE_TYPE_ASIC_VDDC);
    981 	else
    982 		r600_voltage_control_deactivate_static_control(rdev,
    983 							       rv6xx_get_master_voltage_mask(rdev));
    984 }
    985 
    986 static void rv6xx_enable_display_gap(struct radeon_device *rdev, bool enable)
    987 {
    988 	if (enable) {
    989 		u32 tmp = (DISP1_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM) |
    990 			   DISP2_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM) |
    991 			   DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE) |
    992 			   DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE) |
    993 			   VBI_TIMER_COUNT(0x3FFF) |
    994 			   VBI_TIMER_UNIT(7));
    995 		WREG32(CG_DISPLAY_GAP_CNTL, tmp);
    996 
    997 		WREG32_P(MCLK_PWRMGT_CNTL, USE_DISPLAY_GAP, ~USE_DISPLAY_GAP);
    998 	} else
    999 		WREG32_P(MCLK_PWRMGT_CNTL, 0, ~USE_DISPLAY_GAP);
   1000 }
   1001 
   1002 static void rv6xx_program_power_level_enter_state(struct radeon_device *rdev)
   1003 {
   1004 	r600_power_level_set_enter_index(rdev, R600_POWER_LEVEL_MEDIUM);
   1005 }
   1006 
   1007 static void rv6xx_calculate_t(u32 l_f, u32 h_f, int h,
   1008 			      int d_l, int d_r, u8 *l, u8 *r)
   1009 {
   1010 	int a_n, a_d, h_r, l_r;
   1011 
   1012 	h_r = d_l;
   1013 	l_r = 100 - d_r;
   1014 
   1015 	a_n = (int)h_f * d_l + (int)l_f * (h - d_r);
   1016 	a_d = (int)l_f * l_r + (int)h_f * h_r;
   1017 
   1018 	if (a_d != 0) {
   1019 		*l = d_l - h_r * a_n / a_d;
   1020 		*r = d_r + l_r * a_n / a_d;
   1021 	}
   1022 }
   1023 
   1024 static void rv6xx_calculate_ap(struct radeon_device *rdev,
   1025 			       struct rv6xx_ps *state)
   1026 {
   1027 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1028 
   1029 	pi->hw.lp[0] = 0;
   1030 	pi->hw.rp[R600_PM_NUMBER_OF_ACTIVITY_LEVELS - 1]
   1031 		= 100;
   1032 
   1033 	rv6xx_calculate_t(state->low.sclk,
   1034 			  state->medium.sclk,
   1035 			  R600_AH_DFLT,
   1036 			  R600_LMP_DFLT,
   1037 			  R600_RLP_DFLT,
   1038 			  &pi->hw.lp[1],
   1039 			  &pi->hw.rp[0]);
   1040 
   1041 	rv6xx_calculate_t(state->medium.sclk,
   1042 			  state->high.sclk,
   1043 			  R600_AH_DFLT,
   1044 			  R600_LHP_DFLT,
   1045 			  R600_RMP_DFLT,
   1046 			  &pi->hw.lp[2],
   1047 			  &pi->hw.rp[1]);
   1048 
   1049 }
   1050 
   1051 static void rv6xx_calculate_stepping_parameters(struct radeon_device *rdev,
   1052 						struct radeon_ps *new_ps)
   1053 {
   1054 	struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
   1055 
   1056 	rv6xx_calculate_engine_speed_stepping_parameters(rdev, new_state);
   1057 	rv6xx_calculate_memory_clock_stepping_parameters(rdev, new_state);
   1058 	rv6xx_calculate_voltage_stepping_parameters(rdev, new_state);
   1059 	rv6xx_calculate_ap(rdev, new_state);
   1060 }
   1061 
   1062 static void rv6xx_program_stepping_parameters_except_lowest_entry(struct radeon_device *rdev)
   1063 {
   1064 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1065 
   1066 	rv6xx_program_mclk_stepping_parameters_except_lowest_entry(rdev);
   1067 	if (pi->voltage_control)
   1068 		rv6xx_program_voltage_stepping_parameters_except_lowest_entry(rdev);
   1069 	rv6xx_program_backbias_stepping_parameters_except_lowest_entry(rdev);
   1070 	rv6xx_program_sclk_spread_spectrum_parameters_except_lowest_entry(rdev);
   1071 	rv6xx_program_mclk_spread_spectrum_parameters(rdev);
   1072 	rv6xx_program_memory_timing_parameters(rdev);
   1073 }
   1074 
   1075 static void rv6xx_program_stepping_parameters_lowest_entry(struct radeon_device *rdev)
   1076 {
   1077 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1078 
   1079 	rv6xx_program_mclk_stepping_parameters_lowest_entry(rdev);
   1080 	if (pi->voltage_control)
   1081 		rv6xx_program_voltage_stepping_parameters_lowest_entry(rdev);
   1082 	rv6xx_program_backbias_stepping_parameters_lowest_entry(rdev);
   1083 	rv6xx_program_sclk_spread_spectrum_parameters_lowest_entry(rdev);
   1084 }
   1085 
   1086 static void rv6xx_program_power_level_low(struct radeon_device *rdev)
   1087 {
   1088 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1089 
   1090 	r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_LOW,
   1091 					   pi->hw.low_vddc_index);
   1092 	r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_LOW,
   1093 					     pi->hw.low_mclk_index);
   1094 	r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_LOW,
   1095 					     pi->hw.low_sclk_index);
   1096 	r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_LOW,
   1097 					  R600_DISPLAY_WATERMARK_LOW);
   1098 	r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_LOW,
   1099 				       pi->hw.pcie_gen2[R600_POWER_LEVEL_LOW]);
   1100 }
   1101 
   1102 static void rv6xx_program_power_level_low_to_lowest_state(struct radeon_device *rdev)
   1103 {
   1104 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1105 
   1106 	r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_LOW, 0);
   1107 	r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_LOW, 0);
   1108 	r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_LOW, 0);
   1109 
   1110 	r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_LOW,
   1111 					  R600_DISPLAY_WATERMARK_LOW);
   1112 
   1113 	r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_LOW,
   1114 				       pi->hw.pcie_gen2[R600_POWER_LEVEL_LOW]);
   1115 
   1116 }
   1117 
   1118 static void rv6xx_program_power_level_medium(struct radeon_device *rdev)
   1119 {
   1120 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1121 
   1122 	r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_MEDIUM,
   1123 					  pi->hw.medium_vddc_index);
   1124 	r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_MEDIUM,
   1125 					    pi->hw.medium_mclk_index);
   1126 	r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_MEDIUM,
   1127 					    pi->hw.medium_sclk_index);
   1128 	r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_MEDIUM,
   1129 					 R600_DISPLAY_WATERMARK_LOW);
   1130 	r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_MEDIUM,
   1131 				      pi->hw.pcie_gen2[R600_POWER_LEVEL_MEDIUM]);
   1132 }
   1133 
   1134 static void rv6xx_program_power_level_medium_for_transition(struct radeon_device *rdev)
   1135 {
   1136 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1137 
   1138 	rv6xx_program_mclk_stepping_entry(rdev,
   1139 					  R600_POWER_LEVEL_CTXSW,
   1140 					  pi->hw.mclks[pi->hw.low_mclk_index]);
   1141 
   1142 	r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_MEDIUM, 1);
   1143 
   1144 	r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_MEDIUM,
   1145 					     R600_POWER_LEVEL_CTXSW);
   1146 	r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_MEDIUM,
   1147 					     pi->hw.medium_sclk_index);
   1148 
   1149 	r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_MEDIUM,
   1150 					  R600_DISPLAY_WATERMARK_LOW);
   1151 
   1152 	rv6xx_enable_engine_spread_spectrum(rdev, R600_POWER_LEVEL_MEDIUM, false);
   1153 
   1154 	r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_MEDIUM,
   1155 				       pi->hw.pcie_gen2[R600_POWER_LEVEL_LOW]);
   1156 }
   1157 
   1158 static void rv6xx_program_power_level_high(struct radeon_device *rdev)
   1159 {
   1160 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1161 
   1162 	r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_HIGH,
   1163 					   pi->hw.high_vddc_index);
   1164 	r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_HIGH,
   1165 					     pi->hw.high_mclk_index);
   1166 	r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_HIGH,
   1167 					     pi->hw.high_sclk_index);
   1168 
   1169 	r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_HIGH,
   1170 					  R600_DISPLAY_WATERMARK_HIGH);
   1171 
   1172 	r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_HIGH,
   1173 				       pi->hw.pcie_gen2[R600_POWER_LEVEL_HIGH]);
   1174 }
   1175 
   1176 static void rv6xx_enable_backbias(struct radeon_device *rdev, bool enable)
   1177 {
   1178 	if (enable)
   1179 		WREG32_P(GENERAL_PWRMGT, BACKBIAS_PAD_EN | BACKBIAS_DPM_CNTL,
   1180 			 ~(BACKBIAS_PAD_EN | BACKBIAS_DPM_CNTL));
   1181 	else
   1182 		WREG32_P(GENERAL_PWRMGT, 0,
   1183 			 ~(BACKBIAS_VALUE | BACKBIAS_PAD_EN | BACKBIAS_DPM_CNTL));
   1184 }
   1185 
   1186 static void rv6xx_program_display_gap(struct radeon_device *rdev)
   1187 {
   1188 	u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL);
   1189 
   1190 	tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK);
   1191 	if (rdev->pm.dpm.new_active_crtcs & 1) {
   1192 		tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK);
   1193 		tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
   1194 	} else if (rdev->pm.dpm.new_active_crtcs & 2) {
   1195 		tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
   1196 		tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK);
   1197 	} else {
   1198 		tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
   1199 		tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
   1200 	}
   1201 	WREG32(CG_DISPLAY_GAP_CNTL, tmp);
   1202 }
   1203 
   1204 static void rv6xx_set_sw_voltage_to_safe(struct radeon_device *rdev,
   1205 					 struct radeon_ps *new_ps,
   1206 					 struct radeon_ps *old_ps)
   1207 {
   1208 	struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
   1209 	struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
   1210 	u16 safe_voltage;
   1211 
   1212 	safe_voltage = (new_state->low.vddc >= old_state->low.vddc) ?
   1213 		new_state->low.vddc : old_state->low.vddc;
   1214 
   1215 	rv6xx_program_voltage_stepping_entry(rdev, R600_POWER_LEVEL_CTXSW,
   1216 					     safe_voltage);
   1217 
   1218 	WREG32_P(GENERAL_PWRMGT, SW_GPIO_INDEX(R600_POWER_LEVEL_CTXSW),
   1219 		 ~SW_GPIO_INDEX_MASK);
   1220 }
   1221 
   1222 static void rv6xx_set_sw_voltage_to_low(struct radeon_device *rdev,
   1223 					struct radeon_ps *old_ps)
   1224 {
   1225 	struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
   1226 
   1227 	rv6xx_program_voltage_stepping_entry(rdev, R600_POWER_LEVEL_CTXSW,
   1228 					     old_state->low.vddc);
   1229 
   1230 	WREG32_P(GENERAL_PWRMGT, SW_GPIO_INDEX(R600_POWER_LEVEL_CTXSW),
   1231 		~SW_GPIO_INDEX_MASK);
   1232 }
   1233 
   1234 static void rv6xx_set_safe_backbias(struct radeon_device *rdev,
   1235 				    struct radeon_ps *new_ps,
   1236 				    struct radeon_ps *old_ps)
   1237 {
   1238 	struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
   1239 	struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
   1240 
   1241 	if ((new_state->low.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) &&
   1242 	    (old_state->low.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE))
   1243 		WREG32_P(GENERAL_PWRMGT, BACKBIAS_VALUE, ~BACKBIAS_VALUE);
   1244 	else
   1245 		WREG32_P(GENERAL_PWRMGT, 0, ~BACKBIAS_VALUE);
   1246 }
   1247 
   1248 static void rv6xx_set_safe_pcie_gen2(struct radeon_device *rdev,
   1249 				     struct radeon_ps *new_ps,
   1250 				     struct radeon_ps *old_ps)
   1251 {
   1252 	struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
   1253 	struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
   1254 
   1255 	if ((new_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) !=
   1256 	    (old_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2))
   1257 		rv6xx_force_pcie_gen1(rdev);
   1258 }
   1259 
   1260 static void rv6xx_enable_dynamic_voltage_control(struct radeon_device *rdev,
   1261 						 bool enable)
   1262 {
   1263 	if (enable)
   1264 		WREG32_P(GENERAL_PWRMGT, VOLT_PWRMGT_EN, ~VOLT_PWRMGT_EN);
   1265 	else
   1266 		WREG32_P(GENERAL_PWRMGT, 0, ~VOLT_PWRMGT_EN);
   1267 }
   1268 
   1269 static void rv6xx_enable_dynamic_backbias_control(struct radeon_device *rdev,
   1270 						  bool enable)
   1271 {
   1272 	if (enable)
   1273 		WREG32_P(GENERAL_PWRMGT, BACKBIAS_DPM_CNTL, ~BACKBIAS_DPM_CNTL);
   1274 	else
   1275 		WREG32_P(GENERAL_PWRMGT, 0, ~BACKBIAS_DPM_CNTL);
   1276 }
   1277 
   1278 static int rv6xx_step_sw_voltage(struct radeon_device *rdev,
   1279 				 u16 initial_voltage,
   1280 				 u16 target_voltage)
   1281 {
   1282 	u16 current_voltage;
   1283 	u16 true_target_voltage;
   1284 	u16 voltage_step;
   1285 	int signed_voltage_step;
   1286 
   1287 	if ((radeon_atom_get_voltage_step(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC,
   1288 					  &voltage_step)) ||
   1289 	    (radeon_atom_round_to_true_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC,
   1290 					       initial_voltage, &current_voltage)) ||
   1291 	    (radeon_atom_round_to_true_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC,
   1292 					       target_voltage, &true_target_voltage)))
   1293 		return -EINVAL;
   1294 
   1295 	if (true_target_voltage < current_voltage)
   1296 		signed_voltage_step = -(int)voltage_step;
   1297 	else
   1298 		signed_voltage_step = voltage_step;
   1299 
   1300 	while (current_voltage != true_target_voltage) {
   1301 		current_voltage += signed_voltage_step;
   1302 		rv6xx_program_voltage_stepping_entry(rdev, R600_POWER_LEVEL_CTXSW,
   1303 						     current_voltage);
   1304 		msleep((rdev->pm.dpm.voltage_response_time + 999) / 1000);
   1305 	}
   1306 
   1307 	return 0;
   1308 }
   1309 
   1310 static int rv6xx_step_voltage_if_increasing(struct radeon_device *rdev,
   1311 					    struct radeon_ps *new_ps,
   1312 					    struct radeon_ps *old_ps)
   1313 {
   1314 	struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
   1315 	struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
   1316 
   1317 	if (new_state->low.vddc > old_state->low.vddc)
   1318 		return rv6xx_step_sw_voltage(rdev,
   1319 					     old_state->low.vddc,
   1320 					     new_state->low.vddc);
   1321 
   1322 	return 0;
   1323 }
   1324 
   1325 static int rv6xx_step_voltage_if_decreasing(struct radeon_device *rdev,
   1326 					    struct radeon_ps *new_ps,
   1327 					    struct radeon_ps *old_ps)
   1328 {
   1329 	struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
   1330 	struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
   1331 
   1332 	if (new_state->low.vddc < old_state->low.vddc)
   1333 		return rv6xx_step_sw_voltage(rdev,
   1334 					     old_state->low.vddc,
   1335 					     new_state->low.vddc);
   1336 	else
   1337 		return 0;
   1338 }
   1339 
   1340 static void rv6xx_enable_high(struct radeon_device *rdev)
   1341 {
   1342 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1343 
   1344 	if ((pi->restricted_levels < 1) ||
   1345 	    (pi->restricted_levels == 3))
   1346 		r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, true);
   1347 }
   1348 
   1349 static void rv6xx_enable_medium(struct radeon_device *rdev)
   1350 {
   1351 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1352 
   1353 	if (pi->restricted_levels < 2)
   1354 		r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, true);
   1355 }
   1356 
   1357 static void rv6xx_set_dpm_event_sources(struct radeon_device *rdev, u32 sources)
   1358 {
   1359 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1360 	bool want_thermal_protection;
   1361 	enum radeon_dpm_event_src dpm_event_src;
   1362 
   1363 	switch (sources) {
   1364         case 0:
   1365         default:
   1366 		want_thermal_protection = false;
   1367 		break;
   1368         case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL):
   1369 		want_thermal_protection = true;
   1370 		dpm_event_src = RADEON_DPM_EVENT_SRC_DIGITAL;
   1371 		break;
   1372 
   1373         case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL):
   1374 		want_thermal_protection = true;
   1375 		dpm_event_src = RADEON_DPM_EVENT_SRC_EXTERNAL;
   1376 		break;
   1377 
   1378         case ((1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL) |
   1379 	      (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL)):
   1380 		want_thermal_protection = true;
   1381 		dpm_event_src = RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL;
   1382 		break;
   1383 	}
   1384 
   1385 	if (want_thermal_protection) {
   1386 		WREG32_P(CG_THERMAL_CTRL, DPM_EVENT_SRC(dpm_event_src), ~DPM_EVENT_SRC_MASK);
   1387 		if (pi->thermal_protection)
   1388 			WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS);
   1389 	} else {
   1390 		WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS);
   1391 	}
   1392 }
   1393 
   1394 static void rv6xx_enable_auto_throttle_source(struct radeon_device *rdev,
   1395 					      enum radeon_dpm_auto_throttle_src source,
   1396 					      bool enable)
   1397 {
   1398 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1399 
   1400 	if (enable) {
   1401 		if (!(pi->active_auto_throttle_sources & (1 << source))) {
   1402 			pi->active_auto_throttle_sources |= 1 << source;
   1403 			rv6xx_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
   1404 		}
   1405 	} else {
   1406 		if (pi->active_auto_throttle_sources & (1 << source)) {
   1407 			pi->active_auto_throttle_sources &= ~(1 << source);
   1408 			rv6xx_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
   1409 		}
   1410 	}
   1411 }
   1412 
   1413 
   1414 static void rv6xx_enable_thermal_protection(struct radeon_device *rdev,
   1415 					    bool enable)
   1416 {
   1417 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1418 
   1419 	if (pi->active_auto_throttle_sources)
   1420 		r600_enable_thermal_protection(rdev, enable);
   1421 }
   1422 
   1423 static void rv6xx_generate_transition_stepping(struct radeon_device *rdev,
   1424 					       struct radeon_ps *new_ps,
   1425 					       struct radeon_ps *old_ps)
   1426 {
   1427 	struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
   1428 	struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
   1429 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1430 
   1431 	rv6xx_generate_steps(rdev,
   1432 			     old_state->low.sclk,
   1433 			     new_state->low.sclk,
   1434 			     0, &pi->hw.medium_sclk_index);
   1435 }
   1436 
   1437 static void rv6xx_generate_low_step(struct radeon_device *rdev,
   1438 				    struct radeon_ps *new_ps)
   1439 {
   1440 	struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
   1441 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1442 
   1443 	pi->hw.low_sclk_index = 0;
   1444 	rv6xx_generate_single_step(rdev,
   1445 				   new_state->low.sclk,
   1446 				   0);
   1447 }
   1448 
   1449 static void rv6xx_invalidate_intermediate_steps(struct radeon_device *rdev)
   1450 {
   1451 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1452 
   1453 	rv6xx_invalidate_intermediate_steps_range(rdev, 0,
   1454 						  pi->hw.medium_sclk_index);
   1455 }
   1456 
   1457 static void rv6xx_generate_stepping_table(struct radeon_device *rdev,
   1458 					  struct radeon_ps *new_ps)
   1459 {
   1460 	struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
   1461 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1462 
   1463 	pi->hw.low_sclk_index = 0;
   1464 
   1465 	rv6xx_generate_steps(rdev,
   1466 			     new_state->low.sclk,
   1467 			     new_state->medium.sclk,
   1468 			     0,
   1469 			     &pi->hw.medium_sclk_index);
   1470 	rv6xx_generate_steps(rdev,
   1471 			     new_state->medium.sclk,
   1472 			     new_state->high.sclk,
   1473 			     pi->hw.medium_sclk_index,
   1474 			     &pi->hw.high_sclk_index);
   1475 }
   1476 
   1477 static void rv6xx_enable_spread_spectrum(struct radeon_device *rdev,
   1478 					 bool enable)
   1479 {
   1480 	if (enable)
   1481 		rv6xx_enable_dynamic_spread_spectrum(rdev, true);
   1482 	else {
   1483 		rv6xx_enable_engine_spread_spectrum(rdev, R600_POWER_LEVEL_LOW, false);
   1484 		rv6xx_enable_engine_spread_spectrum(rdev, R600_POWER_LEVEL_MEDIUM, false);
   1485 		rv6xx_enable_engine_spread_spectrum(rdev, R600_POWER_LEVEL_HIGH, false);
   1486 		rv6xx_enable_dynamic_spread_spectrum(rdev, false);
   1487 		rv6xx_enable_memory_spread_spectrum(rdev, false);
   1488 	}
   1489 }
   1490 
   1491 static void rv6xx_reset_lvtm_data_sync(struct radeon_device *rdev)
   1492 {
   1493 	if (ASIC_IS_DCE3(rdev))
   1494 		WREG32_P(DCE3_LVTMA_DATA_SYNCHRONIZATION, LVTMA_PFREQCHG, ~LVTMA_PFREQCHG);
   1495 	else
   1496 		WREG32_P(LVTMA_DATA_SYNCHRONIZATION, LVTMA_PFREQCHG, ~LVTMA_PFREQCHG);
   1497 }
   1498 
   1499 static void rv6xx_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
   1500 					   struct radeon_ps *new_ps,
   1501 					   bool enable)
   1502 {
   1503 	struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
   1504 
   1505 	if (enable) {
   1506 		rv6xx_enable_bif_dynamic_pcie_gen2(rdev, true);
   1507 		rv6xx_enable_pcie_gen2_support(rdev);
   1508 		r600_enable_dynamic_pcie_gen2(rdev, true);
   1509 	} else {
   1510 		if (!(new_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2))
   1511 			rv6xx_force_pcie_gen1(rdev);
   1512 		rv6xx_enable_bif_dynamic_pcie_gen2(rdev, false);
   1513 		r600_enable_dynamic_pcie_gen2(rdev, false);
   1514 	}
   1515 }
   1516 
   1517 static void rv6xx_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
   1518 						     struct radeon_ps *new_ps,
   1519 						     struct radeon_ps *old_ps)
   1520 {
   1521 	struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
   1522 	struct rv6xx_ps *current_state = rv6xx_get_ps(old_ps);
   1523 
   1524 	if ((new_ps->vclk == old_ps->vclk) &&
   1525 	    (new_ps->dclk == old_ps->dclk))
   1526 		return;
   1527 
   1528 	if (new_state->high.sclk >= current_state->high.sclk)
   1529 		return;
   1530 
   1531 	radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
   1532 }
   1533 
   1534 static void rv6xx_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
   1535 						    struct radeon_ps *new_ps,
   1536 						    struct radeon_ps *old_ps)
   1537 {
   1538 	struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
   1539 	struct rv6xx_ps *current_state = rv6xx_get_ps(old_ps);
   1540 
   1541 	if ((new_ps->vclk == old_ps->vclk) &&
   1542 	    (new_ps->dclk == old_ps->dclk))
   1543 		return;
   1544 
   1545 	if (new_state->high.sclk < current_state->high.sclk)
   1546 		return;
   1547 
   1548 	radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
   1549 }
   1550 
   1551 int rv6xx_dpm_enable(struct radeon_device *rdev)
   1552 {
   1553 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1554 	struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
   1555 
   1556 	if (r600_dynamicpm_enabled(rdev))
   1557 		return -EINVAL;
   1558 
   1559 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
   1560 		rv6xx_enable_backbias(rdev, true);
   1561 
   1562 	if (pi->dynamic_ss)
   1563 		rv6xx_enable_spread_spectrum(rdev, true);
   1564 
   1565 	rv6xx_program_mpll_timing_parameters(rdev);
   1566 	rv6xx_program_bsp(rdev);
   1567 	rv6xx_program_git(rdev);
   1568 	rv6xx_program_tp(rdev);
   1569 	rv6xx_program_tpp(rdev);
   1570 	rv6xx_program_sstp(rdev);
   1571 	rv6xx_program_fcp(rdev);
   1572 	rv6xx_program_vddc3d_parameters(rdev);
   1573 	rv6xx_program_voltage_timing_parameters(rdev);
   1574 	rv6xx_program_engine_speed_parameters(rdev);
   1575 
   1576 	rv6xx_enable_display_gap(rdev, true);
   1577 	if (pi->display_gap == false)
   1578 		rv6xx_enable_display_gap(rdev, false);
   1579 
   1580 	rv6xx_program_power_level_enter_state(rdev);
   1581 
   1582 	rv6xx_calculate_stepping_parameters(rdev, boot_ps);
   1583 
   1584 	if (pi->voltage_control)
   1585 		rv6xx_program_voltage_gpio_pins(rdev);
   1586 
   1587 	rv6xx_generate_stepping_table(rdev, boot_ps);
   1588 
   1589 	rv6xx_program_stepping_parameters_except_lowest_entry(rdev);
   1590 	rv6xx_program_stepping_parameters_lowest_entry(rdev);
   1591 
   1592 	rv6xx_program_power_level_low(rdev);
   1593 	rv6xx_program_power_level_medium(rdev);
   1594 	rv6xx_program_power_level_high(rdev);
   1595 	rv6xx_program_vc(rdev);
   1596 	rv6xx_program_at(rdev);
   1597 
   1598 	r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true);
   1599 	r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, true);
   1600 	r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, true);
   1601 
   1602 	rv6xx_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
   1603 
   1604 	r600_start_dpm(rdev);
   1605 
   1606 	if (pi->voltage_control)
   1607 		rv6xx_enable_static_voltage_control(rdev, boot_ps, false);
   1608 
   1609 	if (pi->dynamic_pcie_gen2)
   1610 		rv6xx_enable_dynamic_pcie_gen2(rdev, boot_ps, true);
   1611 
   1612 	if (pi->gfx_clock_gating)
   1613 		r600_gfx_clockgating_enable(rdev, true);
   1614 
   1615 	return 0;
   1616 }
   1617 
   1618 void rv6xx_dpm_disable(struct radeon_device *rdev)
   1619 {
   1620 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1621 	struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
   1622 
   1623 	if (!r600_dynamicpm_enabled(rdev))
   1624 		return;
   1625 
   1626 	r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true);
   1627 	r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, true);
   1628 	rv6xx_enable_display_gap(rdev, false);
   1629 	rv6xx_clear_vc(rdev);
   1630 	r600_set_at(rdev, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF);
   1631 
   1632 	if (pi->thermal_protection)
   1633 		r600_enable_thermal_protection(rdev, false);
   1634 
   1635 	r600_wait_for_power_level(rdev, R600_POWER_LEVEL_LOW);
   1636 	r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, false);
   1637 	r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false);
   1638 
   1639 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
   1640 		rv6xx_enable_backbias(rdev, false);
   1641 
   1642 	rv6xx_enable_spread_spectrum(rdev, false);
   1643 
   1644 	if (pi->voltage_control)
   1645 		rv6xx_enable_static_voltage_control(rdev, boot_ps, true);
   1646 
   1647 	if (pi->dynamic_pcie_gen2)
   1648 		rv6xx_enable_dynamic_pcie_gen2(rdev, boot_ps, false);
   1649 
   1650 	if (rdev->irq.installed &&
   1651 	    r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
   1652 		rdev->irq.dpm_thermal = false;
   1653 		radeon_irq_set(rdev);
   1654 	}
   1655 
   1656 	if (pi->gfx_clock_gating)
   1657 		r600_gfx_clockgating_enable(rdev, false);
   1658 
   1659 	r600_stop_dpm(rdev);
   1660 }
   1661 
   1662 int rv6xx_dpm_set_power_state(struct radeon_device *rdev)
   1663 {
   1664 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   1665 	struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
   1666 	struct radeon_ps *old_ps = rdev->pm.dpm.current_ps;
   1667 	int ret;
   1668 
   1669 	pi->restricted_levels = 0;
   1670 
   1671 	rv6xx_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
   1672 
   1673 	rv6xx_clear_vc(rdev);
   1674 	r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true);
   1675 	r600_set_at(rdev, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF);
   1676 
   1677 	if (pi->thermal_protection)
   1678 		r600_enable_thermal_protection(rdev, false);
   1679 
   1680 	r600_wait_for_power_level(rdev, R600_POWER_LEVEL_LOW);
   1681 	r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, false);
   1682 	r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false);
   1683 
   1684 	rv6xx_generate_transition_stepping(rdev, new_ps, old_ps);
   1685 	rv6xx_program_power_level_medium_for_transition(rdev);
   1686 
   1687 	if (pi->voltage_control) {
   1688 		rv6xx_set_sw_voltage_to_safe(rdev, new_ps, old_ps);
   1689 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
   1690 			rv6xx_set_sw_voltage_to_low(rdev, old_ps);
   1691 	}
   1692 
   1693 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
   1694 		rv6xx_set_safe_backbias(rdev, new_ps, old_ps);
   1695 
   1696 	if (pi->dynamic_pcie_gen2)
   1697 		rv6xx_set_safe_pcie_gen2(rdev, new_ps, old_ps);
   1698 
   1699 	if (pi->voltage_control)
   1700 		rv6xx_enable_dynamic_voltage_control(rdev, false);
   1701 
   1702 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
   1703 		rv6xx_enable_dynamic_backbias_control(rdev, false);
   1704 
   1705 	if (pi->voltage_control) {
   1706 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
   1707 			rv6xx_step_voltage_if_increasing(rdev, new_ps, old_ps);
   1708 		msleep((rdev->pm.dpm.voltage_response_time + 999) / 1000);
   1709 	}
   1710 
   1711 	r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, true);
   1712 	r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, false);
   1713 	r600_wait_for_power_level_unequal(rdev, R600_POWER_LEVEL_LOW);
   1714 
   1715 	rv6xx_generate_low_step(rdev, new_ps);
   1716 	rv6xx_invalidate_intermediate_steps(rdev);
   1717 	rv6xx_calculate_stepping_parameters(rdev, new_ps);
   1718 	rv6xx_program_stepping_parameters_lowest_entry(rdev);
   1719 	rv6xx_program_power_level_low_to_lowest_state(rdev);
   1720 
   1721 	r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true);
   1722 	r600_wait_for_power_level(rdev, R600_POWER_LEVEL_LOW);
   1723 	r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false);
   1724 
   1725 	if (pi->voltage_control) {
   1726 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC) {
   1727 			ret = rv6xx_step_voltage_if_decreasing(rdev, new_ps, old_ps);
   1728 			if (ret)
   1729 				return ret;
   1730 		}
   1731 		rv6xx_enable_dynamic_voltage_control(rdev, true);
   1732 	}
   1733 
   1734 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
   1735 		rv6xx_enable_dynamic_backbias_control(rdev, true);
   1736 
   1737 	if (pi->dynamic_pcie_gen2)
   1738 		rv6xx_enable_dynamic_pcie_gen2(rdev, new_ps, true);
   1739 
   1740 	rv6xx_reset_lvtm_data_sync(rdev);
   1741 
   1742 	rv6xx_generate_stepping_table(rdev, new_ps);
   1743 	rv6xx_program_stepping_parameters_except_lowest_entry(rdev);
   1744 	rv6xx_program_power_level_low(rdev);
   1745 	rv6xx_program_power_level_medium(rdev);
   1746 	rv6xx_program_power_level_high(rdev);
   1747 	rv6xx_enable_medium(rdev);
   1748 	rv6xx_enable_high(rdev);
   1749 
   1750 	if (pi->thermal_protection)
   1751 		rv6xx_enable_thermal_protection(rdev, true);
   1752 	rv6xx_program_vc(rdev);
   1753 	rv6xx_program_at(rdev);
   1754 
   1755 	rv6xx_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
   1756 
   1757 	return 0;
   1758 }
   1759 
   1760 void rv6xx_setup_asic(struct radeon_device *rdev)
   1761 {
   1762 	r600_enable_acpi_pm(rdev);
   1763 
   1764 	if (radeon_aspm != 0) {
   1765 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L0s)
   1766 			rv6xx_enable_l0s(rdev);
   1767 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L1)
   1768 			rv6xx_enable_l1(rdev);
   1769 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_TURNOFFPLL_ASPML1)
   1770 			rv6xx_enable_pll_sleep_in_l1(rdev);
   1771 	}
   1772 }
   1773 
   1774 void rv6xx_dpm_display_configuration_changed(struct radeon_device *rdev)
   1775 {
   1776 	rv6xx_program_display_gap(rdev);
   1777 }
   1778 
   1779 union power_info {
   1780 	struct _ATOM_POWERPLAY_INFO info;
   1781 	struct _ATOM_POWERPLAY_INFO_V2 info_2;
   1782 	struct _ATOM_POWERPLAY_INFO_V3 info_3;
   1783 	struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
   1784 	struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
   1785 	struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
   1786 };
   1787 
   1788 union pplib_clock_info {
   1789 	struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
   1790 	struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
   1791 	struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
   1792 	struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
   1793 };
   1794 
   1795 union pplib_power_state {
   1796 	struct _ATOM_PPLIB_STATE v1;
   1797 	struct _ATOM_PPLIB_STATE_V2 v2;
   1798 };
   1799 
   1800 static void rv6xx_parse_pplib_non_clock_info(struct radeon_device *rdev,
   1801 					     struct radeon_ps *rps,
   1802 					     struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info)
   1803 {
   1804 	rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
   1805 	rps->class = le16_to_cpu(non_clock_info->usClassification);
   1806 	rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
   1807 
   1808 	if (r600_is_uvd_state(rps->class, rps->class2)) {
   1809 		rps->vclk = RV6XX_DEFAULT_VCLK_FREQ;
   1810 		rps->dclk = RV6XX_DEFAULT_DCLK_FREQ;
   1811 	} else {
   1812 		rps->vclk = 0;
   1813 		rps->dclk = 0;
   1814 	}
   1815 
   1816 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
   1817 		rdev->pm.dpm.boot_ps = rps;
   1818 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
   1819 		rdev->pm.dpm.uvd_ps = rps;
   1820 }
   1821 
   1822 static void rv6xx_parse_pplib_clock_info(struct radeon_device *rdev,
   1823 					 struct radeon_ps *rps, int index,
   1824 					 union pplib_clock_info *clock_info)
   1825 {
   1826 	struct rv6xx_ps *ps = rv6xx_get_ps(rps);
   1827 	u32 sclk, mclk;
   1828 	u16 vddc;
   1829 	struct rv6xx_pl *pl;
   1830 
   1831 	switch (index) {
   1832 	case 0:
   1833 		pl = &ps->low;
   1834 		break;
   1835 	case 1:
   1836 		pl = &ps->medium;
   1837 		break;
   1838 	case 2:
   1839 	default:
   1840 		pl = &ps->high;
   1841 		break;
   1842 	}
   1843 
   1844 	sclk = le16_to_cpu(clock_info->r600.usEngineClockLow);
   1845 	sclk |= clock_info->r600.ucEngineClockHigh << 16;
   1846 	mclk = le16_to_cpu(clock_info->r600.usMemoryClockLow);
   1847 	mclk |= clock_info->r600.ucMemoryClockHigh << 16;
   1848 
   1849 	pl->mclk = mclk;
   1850 	pl->sclk = sclk;
   1851 	pl->vddc = le16_to_cpu(clock_info->r600.usVDDC);
   1852 	pl->flags = le32_to_cpu(clock_info->r600.ulFlags);
   1853 
   1854 	/* patch up vddc if necessary */
   1855 	if (pl->vddc == 0xff01) {
   1856 		if (radeon_atom_get_max_vddc(rdev, 0, 0, &vddc) == 0)
   1857 			pl->vddc = vddc;
   1858 	}
   1859 
   1860 	/* fix up pcie gen2 */
   1861 	if (pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) {
   1862 		if ((rdev->family == CHIP_RV610) || (rdev->family == CHIP_RV630)) {
   1863 			if (pl->vddc < 1100)
   1864 				pl->flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
   1865 		}
   1866 	}
   1867 
   1868 	/* patch up boot state */
   1869 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
   1870 		u16 vddc, vddci, mvdd;
   1871 		radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
   1872 		pl->mclk = rdev->clock.default_mclk;
   1873 		pl->sclk = rdev->clock.default_sclk;
   1874 		pl->vddc = vddc;
   1875 	}
   1876 }
   1877 
   1878 static int rv6xx_parse_power_table(struct radeon_device *rdev)
   1879 {
   1880 	struct radeon_mode_info *mode_info = &rdev->mode_info;
   1881 	struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
   1882 	union pplib_power_state *power_state;
   1883 	int i, j;
   1884 	union pplib_clock_info *clock_info;
   1885 	union power_info *power_info;
   1886 	int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
   1887         u16 data_offset;
   1888 	u8 frev, crev;
   1889 	struct rv6xx_ps *ps;
   1890 
   1891 	if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
   1892 				   &frev, &crev, &data_offset))
   1893 		return -EINVAL;
   1894 	power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
   1895 
   1896 	rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
   1897 				  power_info->pplib.ucNumStates, GFP_KERNEL);
   1898 	if (!rdev->pm.dpm.ps)
   1899 		return -ENOMEM;
   1900 
   1901 	for (i = 0; i < power_info->pplib.ucNumStates; i++) {
   1902 		power_state = (union pplib_power_state *)
   1903 			(mode_info->atom_context->bios + data_offset +
   1904 			 le16_to_cpu(power_info->pplib.usStateArrayOffset) +
   1905 			 i * power_info->pplib.ucStateEntrySize);
   1906 		non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
   1907 			(mode_info->atom_context->bios + data_offset +
   1908 			 le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
   1909 			 (power_state->v1.ucNonClockStateIndex *
   1910 			  power_info->pplib.ucNonClockSize));
   1911 		if (power_info->pplib.ucStateEntrySize - 1) {
   1912 			u8 *idx;
   1913 			ps = kzalloc(sizeof(struct rv6xx_ps), GFP_KERNEL);
   1914 			if (ps == NULL) {
   1915 				kfree(rdev->pm.dpm.ps);
   1916 				return -ENOMEM;
   1917 			}
   1918 			rdev->pm.dpm.ps[i].ps_priv = ps;
   1919 			rv6xx_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
   1920 							 non_clock_info);
   1921 			idx = (u8 *)&power_state->v1.ucClockStateIndices[0];
   1922 			for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
   1923 				clock_info = (union pplib_clock_info *)
   1924 					(mode_info->atom_context->bios + data_offset +
   1925 					 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
   1926 					 (idx[j] * power_info->pplib.ucClockInfoSize));
   1927 				rv6xx_parse_pplib_clock_info(rdev,
   1928 							     &rdev->pm.dpm.ps[i], j,
   1929 							     clock_info);
   1930 			}
   1931 		}
   1932 	}
   1933 	rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
   1934 	return 0;
   1935 }
   1936 
   1937 int rv6xx_dpm_init(struct radeon_device *rdev)
   1938 {
   1939 	struct radeon_atom_ss ss;
   1940 	struct atom_clock_dividers dividers;
   1941 	struct rv6xx_power_info *pi;
   1942 	int ret;
   1943 
   1944 	pi = kzalloc(sizeof(struct rv6xx_power_info), GFP_KERNEL);
   1945 	if (pi == NULL)
   1946 		return -ENOMEM;
   1947 	rdev->pm.dpm.priv = pi;
   1948 
   1949 	ret = r600_get_platform_caps(rdev);
   1950 	if (ret)
   1951 		return ret;
   1952 
   1953 	ret = rv6xx_parse_power_table(rdev);
   1954 	if (ret)
   1955 		return ret;
   1956 
   1957 	if (rdev->pm.dpm.voltage_response_time == 0)
   1958 		rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
   1959 	if (rdev->pm.dpm.backbias_response_time == 0)
   1960 		rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
   1961 
   1962 	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
   1963 					     0, false, &dividers);
   1964 	if (ret)
   1965 		pi->spll_ref_div = dividers.ref_div + 1;
   1966 	else
   1967 		pi->spll_ref_div = R600_REFERENCEDIVIDER_DFLT;
   1968 
   1969 	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
   1970 					     0, false, &dividers);
   1971 	if (ret)
   1972 		pi->mpll_ref_div = dividers.ref_div + 1;
   1973 	else
   1974 		pi->mpll_ref_div = R600_REFERENCEDIVIDER_DFLT;
   1975 
   1976 	if (rdev->family >= CHIP_RV670)
   1977 		pi->fb_div_scale = 1;
   1978 	else
   1979 		pi->fb_div_scale = 0;
   1980 
   1981 	pi->voltage_control =
   1982 		radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
   1983 
   1984 	pi->gfx_clock_gating = true;
   1985 
   1986 	pi->sclk_ss = radeon_atombios_get_asic_ss_info(rdev, &ss,
   1987 						       ASIC_INTERNAL_ENGINE_SS, 0);
   1988 	pi->mclk_ss = radeon_atombios_get_asic_ss_info(rdev, &ss,
   1989 						       ASIC_INTERNAL_MEMORY_SS, 0);
   1990 
   1991 	/* Disable sclk ss, causes hangs on a lot of systems */
   1992 	pi->sclk_ss = false;
   1993 
   1994 	if (pi->sclk_ss || pi->mclk_ss)
   1995 		pi->dynamic_ss = true;
   1996 	else
   1997 		pi->dynamic_ss = false;
   1998 
   1999 	pi->dynamic_pcie_gen2 = true;
   2000 
   2001 	if (pi->gfx_clock_gating &&
   2002 	    (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE))
   2003 		pi->thermal_protection = true;
   2004 	else
   2005 		pi->thermal_protection = false;
   2006 
   2007 	pi->display_gap = true;
   2008 
   2009 	return 0;
   2010 }
   2011 
   2012 void rv6xx_dpm_print_power_state(struct radeon_device *rdev,
   2013 				 struct radeon_ps *rps)
   2014 {
   2015 	struct rv6xx_ps *ps = rv6xx_get_ps(rps);
   2016 	struct rv6xx_pl *pl;
   2017 
   2018 	r600_dpm_print_class_info(rps->class, rps->class2);
   2019 	r600_dpm_print_cap_info(rps->caps);
   2020 	printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
   2021 	pl = &ps->low;
   2022 	printk("\t\tpower level 0    sclk: %u mclk: %u vddc: %u\n",
   2023 	       pl->sclk, pl->mclk, pl->vddc);
   2024 	pl = &ps->medium;
   2025 	printk("\t\tpower level 1    sclk: %u mclk: %u vddc: %u\n",
   2026 	       pl->sclk, pl->mclk, pl->vddc);
   2027 	pl = &ps->high;
   2028 	printk("\t\tpower level 2    sclk: %u mclk: %u vddc: %u\n",
   2029 	       pl->sclk, pl->mclk, pl->vddc);
   2030 	r600_dpm_print_ps_status(rdev, rps);
   2031 }
   2032 
   2033 #ifdef CONFIG_DEBUG_FS
   2034 void rv6xx_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
   2035 						       struct seq_file *m)
   2036 {
   2037 	struct radeon_ps *rps = rdev->pm.dpm.current_ps;
   2038 	struct rv6xx_ps *ps = rv6xx_get_ps(rps);
   2039 	struct rv6xx_pl *pl;
   2040 	u32 current_index =
   2041 		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >>
   2042 		CURRENT_PROFILE_INDEX_SHIFT;
   2043 
   2044 	if (current_index > 2) {
   2045 		seq_printf(m, "invalid dpm profile %d\n", current_index);
   2046 	} else {
   2047 		if (current_index == 0)
   2048 			pl = &ps->low;
   2049 		else if (current_index == 1)
   2050 			pl = &ps->medium;
   2051 		else /* current_index == 2 */
   2052 			pl = &ps->high;
   2053 		seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
   2054 		seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u\n",
   2055 			   current_index, pl->sclk, pl->mclk, pl->vddc);
   2056 	}
   2057 }
   2058 #endif
   2059 
   2060 /* get the current sclk in 10 khz units */
   2061 u32 rv6xx_dpm_get_current_sclk(struct radeon_device *rdev)
   2062 {
   2063 	struct radeon_ps *rps = rdev->pm.dpm.current_ps;
   2064 	struct rv6xx_ps *ps = rv6xx_get_ps(rps);
   2065 	struct rv6xx_pl *pl;
   2066 	u32 current_index =
   2067 		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >>
   2068 		CURRENT_PROFILE_INDEX_SHIFT;
   2069 
   2070 	if (current_index > 2) {
   2071 		return 0;
   2072 	} else {
   2073 		if (current_index == 0)
   2074 			pl = &ps->low;
   2075 		else if (current_index == 1)
   2076 			pl = &ps->medium;
   2077 		else /* current_index == 2 */
   2078 			pl = &ps->high;
   2079 		return pl->sclk;
   2080 	}
   2081 }
   2082 
   2083 /* get the current mclk in 10 khz units */
   2084 u32 rv6xx_dpm_get_current_mclk(struct radeon_device *rdev)
   2085 {
   2086 	struct radeon_ps *rps = rdev->pm.dpm.current_ps;
   2087 	struct rv6xx_ps *ps = rv6xx_get_ps(rps);
   2088 	struct rv6xx_pl *pl;
   2089 	u32 current_index =
   2090 		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >>
   2091 		CURRENT_PROFILE_INDEX_SHIFT;
   2092 
   2093 	if (current_index > 2) {
   2094 		return 0;
   2095 	} else {
   2096 		if (current_index == 0)
   2097 			pl = &ps->low;
   2098 		else if (current_index == 1)
   2099 			pl = &ps->medium;
   2100 		else /* current_index == 2 */
   2101 			pl = &ps->high;
   2102 		return pl->mclk;
   2103 	}
   2104 }
   2105 
   2106 void rv6xx_dpm_fini(struct radeon_device *rdev)
   2107 {
   2108 	int i;
   2109 
   2110 	for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
   2111 		kfree(rdev->pm.dpm.ps[i].ps_priv);
   2112 	}
   2113 	kfree(rdev->pm.dpm.ps);
   2114 	kfree(rdev->pm.dpm.priv);
   2115 }
   2116 
   2117 u32 rv6xx_dpm_get_sclk(struct radeon_device *rdev, bool low)
   2118 {
   2119 	struct rv6xx_ps *requested_state = rv6xx_get_ps(rdev->pm.dpm.requested_ps);
   2120 
   2121 	if (low)
   2122 		return requested_state->low.sclk;
   2123 	else
   2124 		return requested_state->high.sclk;
   2125 }
   2126 
   2127 u32 rv6xx_dpm_get_mclk(struct radeon_device *rdev, bool low)
   2128 {
   2129 	struct rv6xx_ps *requested_state = rv6xx_get_ps(rdev->pm.dpm.requested_ps);
   2130 
   2131 	if (low)
   2132 		return requested_state->low.mclk;
   2133 	else
   2134 		return requested_state->high.mclk;
   2135 }
   2136 
   2137 int rv6xx_dpm_force_performance_level(struct radeon_device *rdev,
   2138 				      enum radeon_dpm_forced_level level)
   2139 {
   2140 	struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
   2141 
   2142 	if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
   2143 		pi->restricted_levels = 3;
   2144 	} else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
   2145 		pi->restricted_levels = 2;
   2146 	} else {
   2147 		pi->restricted_levels = 0;
   2148 	}
   2149 
   2150 	rv6xx_clear_vc(rdev);
   2151 	r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true);
   2152 	r600_set_at(rdev, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF);
   2153 	r600_wait_for_power_level(rdev, R600_POWER_LEVEL_LOW);
   2154 	r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, false);
   2155 	r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false);
   2156 	rv6xx_enable_medium(rdev);
   2157 	rv6xx_enable_high(rdev);
   2158 	if (pi->restricted_levels == 3)
   2159 		r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, false);
   2160 	rv6xx_program_vc(rdev);
   2161 	rv6xx_program_at(rdev);
   2162 
   2163 	rdev->pm.dpm.forced_level = level;
   2164 
   2165 	return 0;
   2166 }
   2167