intel_pm.c 224 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
/*
 * Copyright © 2012 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eugeni Dodonov <eugeni.dodonov@intel.com>
 *
 */

28
#include <linux/module.h>
29
#include <linux/pm_runtime.h>
30 31 32

#include <drm/drm_atomic_helper.h>
#include <drm/drm_fourcc.h>
33
#include <drm/drm_plane_helper.h>
34

35
#include "display/intel_atomic.h"
36
#include "display/intel_display_types.h"
37 38 39
#include "display/intel_fbc.h"
#include "display/intel_sprite.h"

40 41
#include "gt/intel_llc.h"

42
#include "i915_drv.h"
43
#include "i915_fixed.h"
44
#include "i915_irq.h"
45
#include "i915_trace.h"
46
#include "display/intel_bw.h"
47
#include "intel_pm.h"
48
#include "intel_sideband.h"
49
#include "../../../platform/x86/intel_ips.h"
50

51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
/* Stores plane specific WM parameters */
struct skl_wm_params {
	bool x_tiled, y_tiled;
	bool rc_surface;
	bool is_planar;
	u32 width;
	u8 cpp;
	u32 plane_pixel_rate;
	u32 y_min_scanlines;
	u32 plane_bytes_per_line;
	uint_fixed_16_16_t plane_blocks_per_line;
	uint_fixed_16_16_t y_tile_minimum;
	u32 linetime_us;
	u32 dbuf_block_size;
};

/* used in computing the new watermarks state */
struct intel_wm_config {
	unsigned int num_pipes_active;
	bool sprites_enabled;
	bool sprites_scaled;
};

74
static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
75
{
76 77 78
	if (HAS_LLC(dev_priv)) {
		/*
		 * WaCompressedResourceDisplayNewHashMode:skl,kbl
79
		 * Display WA #0390: skl,kbl
80 81 82 83 84 85 86 87 88
		 *
		 * Must match Sampler, Pixel Back End, and Media. See
		 * WaCompressedResourceSamplerPbeMediaNewHashMode.
		 */
		I915_WRITE(CHICKEN_PAR1_1,
			   I915_READ(CHICKEN_PAR1_1) |
			   SKL_DE_COMPRESSED_HASH_MODE);
	}

89
	/* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
90 91 92
	I915_WRITE(CHICKEN_PAR1_1,
		   I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);

93
	/* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
94 95
	I915_WRITE(GEN8_CHICKEN_DCPR_1,
		   I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
96

97 98
	/* WaFbcTurnOffFbcWatermark:skl,bxt,kbl,cfl */
	/* WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl */
99 100 101
	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
		   DISP_FBC_WM_DIS |
		   DISP_FBC_MEMORY_WAKE);
102

103
	/* WaFbcHighMemBwCorruptionAvoidance:skl,bxt,kbl,cfl */
104 105
	I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
		   ILK_DPFC_DISABLE_DUMMY0);
106 107 108 109 110 111

	if (IS_SKYLAKE(dev_priv)) {
		/* WaDisableDopClockGating */
		I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
			   & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	}
112 113
}

114
static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
115
{
116
	gen9_init_clock_gating(dev_priv);
117

118 119 120 121
	/* WaDisableSDEUnitClockGating:bxt */
	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
		   GEN8_SDEUNIT_CLOCK_GATE_DISABLE);

122 123
	/*
	 * FIXME:
124
	 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
125 126
	 */
	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
127
		   GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
128 129 130 131 132

	/*
	 * Wa: Backlight PWM may stop in the asserted state, causing backlight
	 * to stay fully on.
	 */
133 134
	I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
		   PWM1_GATING_DIS | PWM2_GATING_DIS);
135 136 137 138 139 140 141 142

	/*
	 * Lower the display internal timeout.
	 * This is needed to avoid any hard hangs when DSI port PLL
	 * is off and a MMIO access is attempted by any privilege
	 * application, using batch buffers or any other means.
	 */
	I915_WRITE(RM_TIMEOUT, MMIO_TIMEOUT_US(950));
143 144
}

145 146 147 148 149 150 151 152 153 154 155 156 157
static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
{
	gen9_init_clock_gating(dev_priv);

	/*
	 * WaDisablePWMClockGating:glk
	 * Backlight PWM may stop in the asserted state, causing backlight
	 * to stay fully on.
	 */
	I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
		   PWM1_GATING_DIS | PWM2_GATING_DIS);
}

158
static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
{
	u32 tmp;

	tmp = I915_READ(CLKCFG);

	switch (tmp & CLKCFG_FSB_MASK) {
	case CLKCFG_FSB_533:
		dev_priv->fsb_freq = 533; /* 133*4 */
		break;
	case CLKCFG_FSB_800:
		dev_priv->fsb_freq = 800; /* 200*4 */
		break;
	case CLKCFG_FSB_667:
		dev_priv->fsb_freq =  667; /* 167*4 */
		break;
	case CLKCFG_FSB_400:
		dev_priv->fsb_freq = 400; /* 100*4 */
		break;
	}

	switch (tmp & CLKCFG_MEM_MASK) {
	case CLKCFG_MEM_533:
		dev_priv->mem_freq = 533;
		break;
	case CLKCFG_MEM_667:
		dev_priv->mem_freq = 667;
		break;
	case CLKCFG_MEM_800:
		dev_priv->mem_freq = 800;
		break;
	}

	/* detect pineview DDR3 setting */
	tmp = I915_READ(CSHRDDR3CTL);
	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
}

196
static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
197 198 199
{
	u16 ddrpll, csipll;

200 201
	ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
	csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216

	switch (ddrpll & 0xff) {
	case 0xc:
		dev_priv->mem_freq = 800;
		break;
	case 0x10:
		dev_priv->mem_freq = 1066;
		break;
	case 0x14:
		dev_priv->mem_freq = 1333;
		break;
	case 0x18:
		dev_priv->mem_freq = 1600;
		break;
	default:
217 218
		drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
			ddrpll & 0xff);
219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
		dev_priv->mem_freq = 0;
		break;
	}

	switch (csipll & 0x3ff) {
	case 0x00c:
		dev_priv->fsb_freq = 3200;
		break;
	case 0x00e:
		dev_priv->fsb_freq = 3733;
		break;
	case 0x010:
		dev_priv->fsb_freq = 4266;
		break;
	case 0x012:
		dev_priv->fsb_freq = 4800;
		break;
	case 0x014:
		dev_priv->fsb_freq = 5333;
		break;
	case 0x016:
		dev_priv->fsb_freq = 5866;
		break;
	case 0x018:
		dev_priv->fsb_freq = 6400;
		break;
	default:
246 247
		drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
			csipll & 0x3ff);
248 249 250 251 252
		dev_priv->fsb_freq = 0;
		break;
	}
}

253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
static const struct cxsr_latency cxsr_latency_table[] = {
	{1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
	{1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
	{1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
	{1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
	{1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */

	{1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
	{1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
	{1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
	{1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
	{1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */

	{1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
	{1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
	{1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
	{1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
	{1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */

	{0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
	{0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
	{0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
	{0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
	{0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */

	{0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
	{0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
	{0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
	{0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
	{0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */

	{0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
	{0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
	{0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
	{0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
	{0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
};

291 292
static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
							 bool is_ddr3,
293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
							 int fsb,
							 int mem)
{
	const struct cxsr_latency *latency;
	int i;

	if (fsb == 0 || mem == 0)
		return NULL;

	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		latency = &cxsr_latency_table[i];
		if (is_desktop == latency->is_desktop &&
		    is_ddr3 == latency->is_ddr3 &&
		    fsb == latency->fsb_freq && mem == latency->mem_freq)
			return latency;
	}

	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");

	return NULL;
}

315 316 317 318
static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
{
	u32 val;

319
	vlv_punit_get(dev_priv);
320 321 322 323 324 325 326 327 328 329 330 331

	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
	if (enable)
		val &= ~FORCE_DDR_HIGH_FREQ;
	else
		val |= FORCE_DDR_HIGH_FREQ;
	val &= ~FORCE_DDR_LOW_FREQ;
	val |= FORCE_DDR_FREQ_REQ_ACK;
	vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);

	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
		      FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
332 333
		drm_err(&dev_priv->drm,
			"timed out waiting for Punit DDR DVFS request\n");
334

335
	vlv_punit_put(dev_priv);
336 337
}

338 339 340 341
static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
{
	u32 val;

342
	vlv_punit_get(dev_priv);
343

344
	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
345 346 347 348
	if (enable)
		val |= DSP_MAXFIFO_PM5_ENABLE;
	else
		val &= ~DSP_MAXFIFO_PM5_ENABLE;
349
	vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
350

351
	vlv_punit_put(dev_priv);
352 353
}

354 355 356
#define FW_WM(value, plane) \
	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)

357
static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
358
{
359
	bool was_enabled;
360
	u32 val;
361

362
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
363
		was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
364
		I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
365
		POSTING_READ(FW_BLC_SELF_VLV);
366
	} else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
367
		was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
368
		I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
369
		POSTING_READ(FW_BLC_SELF);
370
	} else if (IS_PINEVIEW(dev_priv)) {
371 372 373 374 375 376
		val = I915_READ(DSPFW3);
		was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
		if (enable)
			val |= PINEVIEW_SELF_REFRESH_EN;
		else
			val &= ~PINEVIEW_SELF_REFRESH_EN;
377
		I915_WRITE(DSPFW3, val);
378
		POSTING_READ(DSPFW3);
379
	} else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
380
		was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
381 382 383
		val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
			       _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
		I915_WRITE(FW_BLC_SELF, val);
384
		POSTING_READ(FW_BLC_SELF);
385
	} else if (IS_I915GM(dev_priv)) {
386 387 388 389 390
		/*
		 * FIXME can't find a bit like this for 915G, and
		 * and yet it does have the related watermark in
		 * FW_BLC_SELF. What's going on?
		 */
391
		was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
392 393 394
		val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
			       _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
		I915_WRITE(INSTPM, val);
395
		POSTING_READ(INSTPM);
396
	} else {
397
		return false;
398
	}
399

400 401
	trace_intel_memory_cxsr(dev_priv, was_enabled, enable);

402 403 404
	drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
		    enableddisabled(enable),
		    enableddisabled(was_enabled));
405 406

	return was_enabled;
407 408
}

Ville Syrjälä's avatar
Ville Syrjälä committed
409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445
/**
 * intel_set_memory_cxsr - Configure CxSR state
 * @dev_priv: i915 device
 * @enable: Allow vs. disallow CxSR
 *
 * Allow or disallow the system to enter a special CxSR
 * (C-state self refresh) state. What typically happens in CxSR mode
 * is that several display FIFOs may get combined into a single larger
 * FIFO for a particular plane (so called max FIFO mode) to allow the
 * system to defer memory fetches longer, and the memory will enter
 * self refresh.
 *
 * Note that enabling CxSR does not guarantee that the system enter
 * this special mode, nor does it guarantee that the system stays
 * in that mode once entered. So this just allows/disallows the system
 * to autonomously utilize the CxSR mode. Other factors such as core
 * C-states will affect when/if the system actually enters/exits the
 * CxSR mode.
 *
 * Note that on VLV/CHV this actually only controls the max FIFO mode,
 * and the system is free to enter/exit memory self refresh at any time
 * even when the use of CxSR has been disallowed.
 *
 * While the system is actually in the CxSR/max FIFO mode, some plane
 * control registers will not get latched on vblank. Thus in order to
 * guarantee the system will respond to changes in the plane registers
 * we must always disallow CxSR prior to making changes to those registers.
 * Unfortunately the system will re-evaluate the CxSR conditions at
 * frame start which happens after vblank start (which is when the plane
 * registers would get latched), so we can't proceed with the plane update
 * during the same frame where we disallowed CxSR.
 *
 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 * the hardware w.r.t. HPLL SR when writing to plane registers.
 * Disallowing just CxSR is sufficient.
 */
446
bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
447
{
448 449
	bool ret;

450
	mutex_lock(&dev_priv->wm.wm_mutex);
451
	ret = _intel_set_memory_cxsr(dev_priv, enable);
452 453 454 455
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
		dev_priv->wm.vlv.cxsr = enable;
	else if (IS_G4X(dev_priv))
		dev_priv->wm.g4x.cxsr = enable;
456
	mutex_unlock(&dev_priv->wm.wm_mutex);
457 458

	return ret;
459
}
460

461 462 463 464 465 466 467 468 469 470 471 472 473 474
/*
 * Latency for FIFO fetches is dependent on several factors:
 *   - memory configuration (speed, channels)
 *   - chipset
 *   - current MCH state
 * It can be fairly high in some situations, so here we assume a fairly
 * pessimal value.  It's a tradeoff between extra memory fetches (if we
 * set this value too high, the FIFO will fetch frequently to stay full)
 * and power consumption (set it too low to save power and we might see
 * FIFO underruns and display "flicker").
 *
 * A value of 5us seems to be a good balance; safe for very low end
 * platforms but not overly aggressive on lower latency configs.
 */
475
static const int pessimal_latency_ns = 5000;
476

477 478 479
#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
	((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))

480
static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
481
{
482
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
483
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
484
	struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
485 486
	enum pipe pipe = crtc->pipe;
	int sprite0_start, sprite1_start;
487
	u32 dsparb, dsparb2, dsparb3;
488

489
	switch (pipe) {
490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508
	case PIPE_A:
		dsparb = I915_READ(DSPARB);
		dsparb2 = I915_READ(DSPARB2);
		sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
		sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
		break;
	case PIPE_B:
		dsparb = I915_READ(DSPARB);
		dsparb2 = I915_READ(DSPARB2);
		sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
		sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
		break;
	case PIPE_C:
		dsparb2 = I915_READ(DSPARB2);
		dsparb3 = I915_READ(DSPARB3);
		sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
		sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
		break;
	default:
509 510
		MISSING_CASE(pipe);
		return;
511 512
	}

513 514 515 516
	fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
	fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
	fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
	fifo_state->plane[PLANE_CURSOR] = 63;
517 518
}

519 520
static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
			      enum i9xx_plane_id i9xx_plane)
521
{
522
	u32 dsparb = I915_READ(DSPARB);
523 524 525
	int size;

	size = dsparb & 0x7f;
526
	if (i9xx_plane == PLANE_B)
527 528
		size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;

529 530
	drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
		    dsparb, plane_name(i9xx_plane), size);
531 532 533 534

	return size;
}

535 536
static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
			      enum i9xx_plane_id i9xx_plane)
537
{
538
	u32 dsparb = I915_READ(DSPARB);
539 540 541
	int size;

	size = dsparb & 0x1ff;
542
	if (i9xx_plane == PLANE_B)
543 544 545
		size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
	size >>= 1; /* Convert to cachelines */

546 547
	drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
		    dsparb, plane_name(i9xx_plane), size);
548 549 550 551

	return size;
}

552 553
static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
			      enum i9xx_plane_id i9xx_plane)
554
{
555
	u32 dsparb = I915_READ(DSPARB);
556 557 558 559 560
	int size;

	size = dsparb & 0x7f;
	size >>= 2; /* Convert to cachelines */

561 562
	drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
		    dsparb, plane_name(i9xx_plane), size);
563 564 565 566 567

	return size;
}

/* Pineview has different values for various configs */
568
static const struct intel_watermark_params pnv_display_wm = {
569 570 571 572 573
	.fifo_size = PINEVIEW_DISPLAY_FIFO,
	.max_wm = PINEVIEW_MAX_WM,
	.default_wm = PINEVIEW_DFT_WM,
	.guard_size = PINEVIEW_GUARD_WM,
	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
574
};
575 576

static const struct intel_watermark_params pnv_display_hplloff_wm = {
577 578 579 580 581
	.fifo_size = PINEVIEW_DISPLAY_FIFO,
	.max_wm = PINEVIEW_MAX_WM,
	.default_wm = PINEVIEW_DFT_HPLLOFF_WM,
	.guard_size = PINEVIEW_GUARD_WM,
	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
582
};
583 584

static const struct intel_watermark_params pnv_cursor_wm = {
585 586 587 588 589
	.fifo_size = PINEVIEW_CURSOR_FIFO,
	.max_wm = PINEVIEW_CURSOR_MAX_WM,
	.default_wm = PINEVIEW_CURSOR_DFT_WM,
	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
590
};
591 592

static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
593 594 595 596 597
	.fifo_size = PINEVIEW_CURSOR_FIFO,
	.max_wm = PINEVIEW_CURSOR_MAX_WM,
	.default_wm = PINEVIEW_CURSOR_DFT_WM,
	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
598
};
599

600
static const struct intel_watermark_params i965_cursor_wm_info = {
601 602 603 604 605
	.fifo_size = I965_CURSOR_FIFO,
	.max_wm = I965_CURSOR_MAX_WM,
	.default_wm = I965_CURSOR_DFT_WM,
	.guard_size = 2,
	.cacheline_size = I915_FIFO_LINE_SIZE,
606
};
607

608
static const struct intel_watermark_params i945_wm_info = {
609 610 611 612 613
	.fifo_size = I945_FIFO_SIZE,
	.max_wm = I915_MAX_WM,
	.default_wm = 1,
	.guard_size = 2,
	.cacheline_size = I915_FIFO_LINE_SIZE,
614
};
615

616
static const struct intel_watermark_params i915_wm_info = {
617 618 619 620 621
	.fifo_size = I915_FIFO_SIZE,
	.max_wm = I915_MAX_WM,
	.default_wm = 1,
	.guard_size = 2,
	.cacheline_size = I915_FIFO_LINE_SIZE,
622
};
623

624
static const struct intel_watermark_params i830_a_wm_info = {
625 626 627 628 629
	.fifo_size = I855GM_FIFO_SIZE,
	.max_wm = I915_MAX_WM,
	.default_wm = 1,
	.guard_size = 2,
	.cacheline_size = I830_FIFO_LINE_SIZE,
630
};
631

632 633 634 635 636 637 638
static const struct intel_watermark_params i830_bc_wm_info = {
	.fifo_size = I855GM_FIFO_SIZE,
	.max_wm = I915_MAX_WM/2,
	.default_wm = 1,
	.guard_size = 2,
	.cacheline_size = I830_FIFO_LINE_SIZE,
};
639

640
static const struct intel_watermark_params i845_wm_info = {
641 642 643 644 645
	.fifo_size = I830_FIFO_SIZE,
	.max_wm = I915_MAX_WM,
	.default_wm = 1,
	.guard_size = 2,
	.cacheline_size = I830_FIFO_LINE_SIZE,
646 647
};

648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684
/**
 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 * @pixel_rate: Pipe pixel rate in kHz
 * @cpp: Plane bytes per pixel
 * @latency: Memory wakeup latency in 0.1us units
 *
 * Compute the watermark using the method 1 or "small buffer"
 * formula. The caller may additonally add extra cachelines
 * to account for TLB misses and clock crossings.
 *
 * This method is concerned with the short term drain rate
 * of the FIFO, ie. it does not account for blanking periods
 * which would effectively reduce the average drain rate across
 * a longer period. The name "small" refers to the fact the
 * FIFO is relatively small compared to the amount of data
 * fetched.
 *
 * The FIFO level vs. time graph might look something like:
 *
 *   |\   |\
 *   | \  | \
 * __---__---__ (- plane active, _ blanking)
 * -> time
 *
 * or perhaps like this:
 *
 *   |\|\  |\|\
 * __----__----__ (- plane active, _ blanking)
 * -> time
 *
 * Returns:
 * The watermark in bytes
 */
static unsigned int intel_wm_method1(unsigned int pixel_rate,
				     unsigned int cpp,
				     unsigned int latency)
{
685
	u64 ret;
686

687
	ret = mul_u32_u32(pixel_rate, cpp * latency);
688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743
	ret = DIV_ROUND_UP_ULL(ret, 10000);

	return ret;
}

/**
 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 * @pixel_rate: Pipe pixel rate in kHz
 * @htotal: Pipe horizontal total
 * @width: Plane width in pixels
 * @cpp: Plane bytes per pixel
 * @latency: Memory wakeup latency in 0.1us units
 *
 * Compute the watermark using the method 2 or "large buffer"
 * formula. The caller may additonally add extra cachelines
 * to account for TLB misses and clock crossings.
 *
 * This method is concerned with the long term drain rate
 * of the FIFO, ie. it does account for blanking periods
 * which effectively reduce the average drain rate across
 * a longer period. The name "large" refers to the fact the
 * FIFO is relatively large compared to the amount of data
 * fetched.
 *
 * The FIFO level vs. time graph might look something like:
 *
 *    |\___       |\___
 *    |    \___   |    \___
 *    |        \  |        \
 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 * -> time
 *
 * Returns:
 * The watermark in bytes
 */
static unsigned int intel_wm_method2(unsigned int pixel_rate,
				     unsigned int htotal,
				     unsigned int width,
				     unsigned int cpp,
				     unsigned int latency)
{
	unsigned int ret;

	/*
	 * FIXME remove once all users are computing
	 * watermarks in the correct place.
	 */
	if (WARN_ON_ONCE(htotal == 0))
		htotal = 1;

	ret = (latency * pixel_rate) / (htotal * 10000);
	ret = (ret + 1) * width * cpp;

	return ret;
}

744 745
/**
 * intel_calculate_wm - calculate watermark level
746
 * @pixel_rate: pixel clock
747
 * @wm: chip FIFO params
748
 * @fifo_size: size of the FIFO buffer
749
 * @cpp: bytes per pixel
750 751 752 753 754 755 756 757 758 759 760 761 762
 * @latency_ns: memory latency for the platform
 *
 * Calculate the watermark level (the level at which the display plane will
 * start fetching from memory again).  Each chip has a different display
 * FIFO size and allocation, so the caller needs to figure that out and pass
 * in the correct intel_watermark_params structure.
 *
 * As the pixel clock runs, the FIFO will be drained at a rate that depends
 * on the pixel size.  When it reaches the watermark level, it'll start
 * fetching FIFO line sized based chunks from memory until the FIFO fills
 * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 * will occur, and a display engine hang could result.
 */
763 764 765 766
static unsigned int intel_calculate_wm(int pixel_rate,
				       const struct intel_watermark_params *wm,
				       int fifo_size, int cpp,
				       unsigned int latency_ns)
767
{
768
	int entries, wm_size;
769 770 771 772 773 774 775

	/*
	 * Note: we need to make sure we don't overflow for various clock &
	 * latency values.
	 * clocks go from a few thousand to several hundred thousand.
	 * latency is usually a few thousand
	 */
776 777 778 779 780
	entries = intel_wm_method1(pixel_rate, cpp,
				   latency_ns / 100);
	entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
		wm->guard_size;
	DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
781

782 783
	wm_size = fifo_size - entries;
	DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
784 785

	/* Don't promote wm_size to unsigned... */
786
	if (wm_size > wm->max_wm)
787 788 789
		wm_size = wm->max_wm;
	if (wm_size <= 0)
		wm_size = wm->default_wm;
790 791 792 793 794 795 796 797 798 799 800

	/*
	 * Bspec seems to indicate that the value shouldn't be lower than
	 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
	 * Lets go for 8 which is the burst size since certain platforms
	 * already use a hardcoded 8 (which is what the spec says should be
	 * done).
	 */
	if (wm_size <= 8)
		wm_size = 8;

801 802 803
	return wm_size;
}

804 805 806 807 808 809 810 811 812 813
static bool is_disabling(int old, int new, int threshold)
{
	return old >= threshold && new < threshold;
}

static bool is_enabling(int old, int new, int threshold)
{
	return old < threshold && new >= threshold;
}

814 815 816 817 818
static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
{
	return dev_priv->wm.max_level + 1;
}

819 820 821
static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
				   const struct intel_plane_state *plane_state)
{
822
	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
823 824

	/* FIXME check the 'enable' instead */
825
	if (!crtc_state->hw.active)
826 827 828 829 830 831 832 833 834 835 836
		return false;

	/*
	 * Treat cursor with fb as always visible since cursor updates
	 * can happen faster than the vrefresh rate, and the current
	 * watermark code doesn't handle that correctly. Cursor updates
	 * which set/clear the fb or change the cursor size are going
	 * to get throttled by intel_legacy_cursor_update() to work
	 * around this problem with the watermark code.
	 */
	if (plane->id == PLANE_CURSOR)
837
		return plane_state->hw.fb != NULL;
838
	else
839
		return plane_state->uapi.visible;
840 841
}

842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860
static bool intel_crtc_active(struct intel_crtc *crtc)
{
	/* Be paranoid as we can arrive here with only partial
	 * state retrieved from the hardware during setup.
	 *
	 * We can ditch the adjusted_mode.crtc_clock check as soon
	 * as Haswell has gained clock readout/fastboot support.
	 *
	 * We can ditch the crtc->primary->state->fb check as soon as we can
	 * properly reconstruct framebuffers.
	 *
	 * FIXME: The intel_crtc->active here should be switched to
	 * crtc->state->active once we have proper CRTC states wired up
	 * for atomic.
	 */
	return crtc->active && crtc->base.primary->state->fb &&
		crtc->config->hw.adjusted_mode.crtc_clock;
}

861
static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
862
{
863
	struct intel_crtc *crtc, *enabled = NULL;
864

865
	for_each_intel_crtc(&dev_priv->drm, crtc) {
866
		if (intel_crtc_active(crtc)) {
867 868 869 870 871 872 873 874 875
			if (enabled)
				return NULL;
			enabled = crtc;
		}
	}

	return enabled;
}

876
static void pnv_update_wm(struct intel_crtc *unused_crtc)
877
{
878
	struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
879
	struct intel_crtc *crtc;
880 881
	const struct cxsr_latency *latency;
	u32 reg;
882
	unsigned int wm;
883

884
	latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
885 886 887
					 dev_priv->is_ddr3,
					 dev_priv->fsb_freq,
					 dev_priv->mem_freq);
888
	if (!latency) {
889 890
		drm_dbg_kms(&dev_priv->drm,
			    "Unknown FSB/MEM found, disable CxSR\n");
891
		intel_set_memory_cxsr(dev_priv, false);
892 893 894
		return;
	}

895
	crtc = single_enabled_crtc(dev_priv);
896
	if (crtc) {
897
		const struct drm_display_mode *adjusted_mode =
898
			&crtc->config->hw.adjusted_mode;
899 900
		const struct drm_framebuffer *fb =
			crtc->base.primary->state->fb;
901
		int cpp = fb->format->cpp[0];
902
		int clock = adjusted_mode->crtc_clock;
903 904

		/* Display SR */
905 906
		wm = intel_calculate_wm(clock, &pnv_display_wm,
					pnv_display_wm.fifo_size,
907
					cpp, latency->display_sr);
908 909
		reg = I915_READ(DSPFW1);
		reg &= ~DSPFW_SR_MASK;
910
		reg |= FW_WM(wm, SR);
911
		I915_WRITE(DSPFW1, reg);
912
		drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
913 914

		/* cursor SR */
915 916
		wm = intel_calculate_wm(clock, &pnv_cursor_wm,
					pnv_display_wm.fifo_size,
917
					4, latency->cursor_sr);
918 919
		reg = I915_READ(DSPFW3);
		reg &= ~DSPFW_CURSOR_SR_MASK;
920
		reg |= FW_WM(wm, CURSOR_SR);
921 922 923
		I915_WRITE(DSPFW3, reg);

		/* Display HPLL off SR */
924 925
		wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
					pnv_display_hplloff_wm.fifo_size,
926
					cpp, latency->display_hpll_disable);
927 928
		reg = I915_READ(DSPFW3);
		reg &= ~DSPFW_HPLL_SR_MASK;
929
		reg |= FW_WM(wm, HPLL_SR);
930 931 932
		I915_WRITE(DSPFW3, reg);

		/* cursor HPLL off SR */
933 934
		wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
					pnv_display_hplloff_wm.fifo_size,
935
					4, latency->cursor_hpll_disable);
936 937
		reg = I915_READ(DSPFW3);
		reg &= ~DSPFW_HPLL_CURSOR_MASK;
938
		reg |= FW_WM(wm, HPLL_CURSOR);
939
		I915_WRITE(DSPFW3, reg);
940
		drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
941

942
		intel_set_memory_cxsr(dev_priv, true);
943
	} else {
944
		intel_set_memory_cxsr(dev_priv, false);
945 946 947
	}
}

948 949 950 951 952 953 954 955 956 957
/*
 * Documentation says:
 * "If the line size is small, the TLB fetches can get in the way of the
 *  data fetches, causing some lag in the pixel data return which is not
 *  accounted for in the above formulas. The following adjustment only
 *  needs to be applied if eight whole lines fit in the buffer at once.
 *  The WM is adjusted upwards by the difference between the FIFO size
 *  and the size of 8 whole lines. This adjustment is always performed
 *  in the actual pixel depth regardless of whether FBC is enabled or not."
 */
958
static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
959 960 961 962 963 964
{
	int tlb_miss = fifo_size * 64 - width * cpp * 8;

	return max(0, tlb_miss);
}

965 966
static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
				const struct g4x_wm_values *wm)
967
{
968 969 970 971 972
	enum pipe pipe;

	for_each_pipe(dev_priv, pipe)
		trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);

973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989
	I915_WRITE(DSPFW1,
		   FW_WM(wm->sr.plane, SR) |
		   FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
		   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
		   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
	I915_WRITE(DSPFW2,
		   (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
		   FW_WM(wm->sr.fbc, FBC_SR) |
		   FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
		   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
		   FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
		   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
	I915_WRITE(DSPFW3,
		   (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
		   FW_WM(wm->sr.cursor, CURSOR_SR) |
		   FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
		   FW_WM(wm->hpll.plane, HPLL_SR));
990

991
	POSTING_READ(DSPFW1);
992 993
}

994 995 996
#define FW_WM_VLV(value, plane) \
	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)

997
static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
998 999
				const struct vlv_wm_values *wm)
{
1000 1001 1002
	enum pipe pipe;

	for_each_pipe(dev_priv, pipe) {
1003 1004
		trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);

1005 1006 1007 1008 1009 1010
		I915_WRITE(VLV_DDL(pipe),
			   (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
			   (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
			   (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
			   (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
	}
1011

1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022
	/*
	 * Zero the (unused) WM1 watermarks, and also clear all the
	 * high order bits so that there are no out of bounds values
	 * present in the registers during the reprogramming.
	 */
	I915_WRITE(DSPHOWM, 0);
	I915_WRITE(DSPHOWM1, 0);
	I915_WRITE(DSPFW4, 0);
	I915_WRITE(DSPFW5, 0);
	I915_WRITE(DSPFW6, 0);

1023
	I915_WRITE(DSPFW1,
1024
		   FW_WM(wm->sr.plane, SR) |
1025 1026 1027
		   FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
		   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
		   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1028
	I915_WRITE(DSPFW2,
1029 1030 1031
		   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
		   FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
		   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1032
	I915_WRITE(DSPFW3,
1033
		   FW_WM(wm->sr.cursor, CURSOR_SR));
1034 1035 1036

	if (IS_CHERRYVIEW(dev_priv)) {
		I915_WRITE(DSPFW7_CHV,
1037 1038
			   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
			   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1039
		I915_WRITE(DSPFW8_CHV,
1040 1041
			   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
			   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1042
		I915_WRITE(DSPFW9_CHV,
1043 1044
			   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
			   FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1045
		I915_WRITE(DSPHOWM,
1046
			   FW_WM(wm->sr.plane >> 9, SR_HI) |
1047 1048 1049 1050 1051 1052 1053 1054 1055
			   FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
			   FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
			   FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1056 1057
	} else {
		I915_WRITE(DSPFW7,
1058 1059
			   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
			   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1060
		I915_WRITE(DSPHOWM,
1061
			   FW_WM(wm->sr.plane >> 9, SR_HI) |
1062 1063 1064 1065 1066 1067
			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1068 1069 1070
	}

	POSTING_READ(DSPFW1);
1071 1072
}

1073 1074
#undef FW_WM_VLV

1075 1076 1077 1078 1079
static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
{
	/* all latencies in usec */
	dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
	dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1080
	dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1081

1082
	dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126
}

static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
{
	/*
	 * DSPCNTR[13] supposedly controls whether the
	 * primary plane can use the FIFO space otherwise
	 * reserved for the sprite plane. It's not 100% clear
	 * what the actual FIFO size is, but it looks like we
	 * can happily set both primary and sprite watermarks
	 * up to 127 cachelines. So that would seem to mean
	 * that either DSPCNTR[13] doesn't do anything, or that
	 * the total FIFO is >= 256 cachelines in size. Either
	 * way, we don't seem to have to worry about this
	 * repartitioning as the maximum watermark value the
	 * register can hold for each plane is lower than the
	 * minimum FIFO size.
	 */
	switch (plane_id) {
	case PLANE_CURSOR:
		return 63;
	case PLANE_PRIMARY:
		return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
	case PLANE_SPRITE0:
		return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
	default:
		MISSING_CASE(plane_id);
		return 0;
	}
}

static int g4x_fbc_fifo_size(int level)
{
	switch (level) {
	case G4X_WM_LEVEL_SR:
		return 7;
	case G4X_WM_LEVEL_HPLL:
		return 15;
	default:
		MISSING_CASE(level);
		return 0;
	}
}

1127 1128 1129
static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
			  const struct intel_plane_state *plane_state,
			  int level)
1130
{
1131
	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1132 1133
	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
	const struct drm_display_mode *adjusted_mode =
1134
		&crtc_state->hw.adjusted_mode;
1135 1136
	unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
	unsigned int clock, htotal, cpp, width, wm;
1137 1138 1139 1140 1141 1142 1143

	if (latency == 0)
		return USHRT_MAX;

	if (!intel_wm_plane_visible(crtc_state, plane_state))
		return 0;

1144
	cpp = plane_state->hw.fb->format->cpp[0];
1145

1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158
	/*
	 * Not 100% sure which way ELK should go here as the
	 * spec only says CL/CTG should assume 32bpp and BW
	 * doesn't need to. But as these things followed the
	 * mobile vs. desktop lines on gen3 as well, let's
	 * assume ELK doesn't need this.
	 *
	 * The spec also fails to list such a restriction for
	 * the HPLL watermark, which seems a little strange.
	 * Let's use 32bpp for the HPLL watermark as well.
	 */
	if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
	    level != G4X_WM_LEVEL_NORMAL)
1159
		cpp = max(cpp, 4u);
1160 1161 1162 1163

	clock = adjusted_mode->crtc_clock;
	htotal = adjusted_mode->crtc_htotal;

1164
	width = drm_rect_width(&plane_state->uapi.dst);
1165 1166 1167 1168 1169 1170 1171

	if (plane->id == PLANE_CURSOR) {
		wm = intel_wm_method2(clock, htotal, width, cpp, latency);
	} else if (plane->id == PLANE_PRIMARY &&
		   level == G4X_WM_LEVEL_NORMAL) {
		wm = intel_wm_method1(clock, cpp, latency);
	} else {
1172
		unsigned int small, large;
1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184

		small = intel_wm_method1(clock, cpp, latency);
		large = intel_wm_method2(clock, htotal, width, cpp, latency);

		wm = min(small, large);
	}

	wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
			      width, cpp);

	wm = DIV_ROUND_UP(wm, 64) + 2;

1185
	return min_t(unsigned int, wm, USHRT_MAX);
1186 1187 1188 1189 1190
}

static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
				 int level, enum plane_id plane_id, u16 value)
{
1191
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206
	bool dirty = false;

	for (; level < intel_wm_num_levels(dev_priv); level++) {
		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];

		dirty |= raw->plane[plane_id] != value;
		raw->plane[plane_id] = value;
	}

	return dirty;
}

static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
			       int level, u16 value)
{
1207
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222
	bool dirty = false;

	/* NORMAL level doesn't have an FBC watermark */
	level = max(level, G4X_WM_LEVEL_SR);

	for (; level < intel_wm_num_levels(dev_priv); level++) {
		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];

		dirty |= raw->fbc != value;
		raw->fbc = value;
	}

	return dirty;
}

1223 1224
static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
			      const struct intel_plane_state *plane_state,
1225
			      u32 pri_val);
1226 1227 1228 1229

static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
				     const struct intel_plane_state *plane_state)
{
1230
	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1231
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283
	int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
	enum plane_id plane_id = plane->id;
	bool dirty = false;
	int level;

	if (!intel_wm_plane_visible(crtc_state, plane_state)) {
		dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
		if (plane_id == PLANE_PRIMARY)
			dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
		goto out;
	}

	for (level = 0; level < num_levels; level++) {
		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
		int wm, max_wm;

		wm = g4x_compute_wm(crtc_state, plane_state, level);
		max_wm = g4x_plane_fifo_size(plane_id, level);

		if (wm > max_wm)
			break;

		dirty |= raw->plane[plane_id] != wm;
		raw->plane[plane_id] = wm;

		if (plane_id != PLANE_PRIMARY ||
		    level == G4X_WM_LEVEL_NORMAL)
			continue;

		wm = ilk_compute_fbc_wm(crtc_state, plane_state,
					raw->plane[plane_id]);
		max_wm = g4x_fbc_fifo_size(level);

		/*
		 * FBC wm is not mandatory as we
		 * can always just disable its use.
		 */
		if (wm > max_wm)
			wm = USHRT_MAX;

		dirty |= raw->fbc != wm;
		raw->fbc = wm;
	}

	/* mark watermarks as invalid */
	dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);

	if (plane_id == PLANE_PRIMARY)
		dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);

 out:
	if (dirty) {
1284 1285 1286 1287 1288 1289
		drm_dbg_kms(&dev_priv->drm,
			    "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
			    plane->base.name,
			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1290 1291

		if (plane_id == PLANE_PRIMARY)
1292 1293 1294 1295
			drm_dbg_kms(&dev_priv->drm,
				    "FBC watermarks: SR=%d, HPLL=%d\n",
				    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
				    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311
	}

	return dirty;
}

static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
				      enum plane_id plane_id, int level)
{
	const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];

	return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
}

static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
				     int level)
{
1312
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349

	if (level > dev_priv->wm.max_level)
		return false;

	return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
		g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
		g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
}

/* mark all levels starting from 'level' as invalid */
static void g4x_invalidate_wms(struct intel_crtc *crtc,
			       struct g4x_wm_state *wm_state, int level)
{
	if (level <= G4X_WM_LEVEL_NORMAL) {
		enum plane_id plane_id;

		for_each_plane_id_on_crtc(crtc, plane_id)
			wm_state->wm.plane[plane_id] = USHRT_MAX;
	}

	if (level <= G4X_WM_LEVEL_SR) {
		wm_state->cxsr = false;
		wm_state->sr.cursor = USHRT_MAX;
		wm_state->sr.plane = USHRT_MAX;
		wm_state->sr.fbc = USHRT_MAX;
	}

	if (level <= G4X_WM_LEVEL_HPLL) {
		wm_state->hpll_en = false;
		wm_state->hpll.cursor = USHRT_MAX;
		wm_state->hpll.plane = USHRT_MAX;
		wm_state->hpll.fbc = USHRT_MAX;
	}
}

static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
{
1350
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1351
	struct intel_atomic_state *state =
1352
		to_intel_atomic_state(crtc_state->uapi.state);
1353
	struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1354 1355
	int num_active_planes = hweight8(crtc_state->active_planes &
					 ~BIT(PLANE_CURSOR));
1356
	const struct g4x_pipe_wm *raw;
1357 1358
	const struct intel_plane_state *old_plane_state;
	const struct intel_plane_state *new_plane_state;
1359 1360 1361 1362 1363
	struct intel_plane *plane;
	enum plane_id plane_id;
	int i, level;
	unsigned int dirty = 0;

1364 1365 1366
	for_each_oldnew_intel_plane_in_state(state, plane,
					     old_plane_state,
					     new_plane_state, i) {
1367 1368
		if (new_plane_state->hw.crtc != &crtc->base &&
		    old_plane_state->hw.crtc != &crtc->base)
1369 1370
			continue;

1371
		if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436
			dirty |= BIT(plane->id);
	}

	if (!dirty)
		return 0;

	level = G4X_WM_LEVEL_NORMAL;
	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
		goto out;

	raw = &crtc_state->wm.g4x.raw[level];
	for_each_plane_id_on_crtc(crtc, plane_id)
		wm_state->wm.plane[plane_id] = raw->plane[plane_id];

	level = G4X_WM_LEVEL_SR;

	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
		goto out;

	raw = &crtc_state->wm.g4x.raw[level];
	wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
	wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
	wm_state->sr.fbc = raw->fbc;

	wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);

	level = G4X_WM_LEVEL_HPLL;

	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
		goto out;

	raw = &crtc_state->wm.g4x.raw[level];
	wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
	wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
	wm_state->hpll.fbc = raw->fbc;

	wm_state->hpll_en = wm_state->cxsr;

	level++;

 out:
	if (level == G4X_WM_LEVEL_NORMAL)
		return -EINVAL;

	/* invalidate the higher levels */
	g4x_invalidate_wms(crtc, wm_state, level);

	/*
	 * Determine if the FBC watermark(s) can be used. IF
	 * this isn't the case we prefer to disable the FBC
	 ( watermark(s) rather than disable the SR/HPLL
	 * level(s) entirely.
	 */
	wm_state->fbc_en = level > G4X_WM_LEVEL_NORMAL;

	if (level >= G4X_WM_LEVEL_SR &&
	    wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
		wm_state->fbc_en = false;
	else if (level >= G4X_WM_LEVEL_HPLL &&
		 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
		wm_state->fbc_en = false;

	return 0;
}

1437
static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
1438
{
1439
	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1440 1441 1442
	struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
	const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
	struct intel_atomic_state *intel_state =
1443
		to_intel_atomic_state(new_crtc_state->uapi.state);
1444 1445 1446
	const struct intel_crtc_state *old_crtc_state =
		intel_atomic_get_old_crtc_state(intel_state, crtc);
	const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1447 1448
	enum plane_id plane_id;

1449
	if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1450 1451 1452 1453 1454 1455 1456
		*intermediate = *optimal;

		intermediate->cxsr = false;
		intermediate->hpll_en = false;
		goto out;
	}

1457
	intermediate->cxsr = optimal->cxsr && active->cxsr &&
1458
		!new_crtc_state->disable_cxsr;
1459
	intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1460
		!new_crtc_state->disable_cxsr;
1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501
	intermediate->fbc_en = optimal->fbc_en && active->fbc_en;

	for_each_plane_id_on_crtc(crtc, plane_id) {
		intermediate->wm.plane[plane_id] =
			max(optimal->wm.plane[plane_id],
			    active->wm.plane[plane_id]);

		WARN_ON(intermediate->wm.plane[plane_id] >
			g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
	}

	intermediate->sr.plane = max(optimal->sr.plane,
				     active->sr.plane);
	intermediate->sr.cursor = max(optimal->sr.cursor,
				      active->sr.cursor);
	intermediate->sr.fbc = max(optimal->sr.fbc,
				   active->sr.fbc);

	intermediate->hpll.plane = max(optimal->hpll.plane,
				       active->hpll.plane);
	intermediate->hpll.cursor = max(optimal->hpll.cursor,
					active->hpll.cursor);
	intermediate->hpll.fbc = max(optimal->hpll.fbc,
				     active->hpll.fbc);

	WARN_ON((intermediate->sr.plane >
		 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
		 intermediate->sr.cursor >
		 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
		intermediate->cxsr);
	WARN_ON((intermediate->sr.plane >
		 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
		 intermediate->sr.cursor >
		 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
		intermediate->hpll_en);

	WARN_ON(intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
		intermediate->fbc_en && intermediate->cxsr);
	WARN_ON(intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
		intermediate->fbc_en && intermediate->hpll_en);

1502
out:
1503 1504 1505 1506 1507
	/*
	 * If our intermediate WM are identical to the final WM, then we can
	 * omit the post-vblank programming; only update if it's different.
	 */
	if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1508
		new_crtc_state->wm.need_postvbl_update = true;
1509 1510 1511 1512 1513 1514 1515 1516

	return 0;
}

static void g4x_merge_wm(struct drm_i915_private *dev_priv,
			 struct g4x_wm_values *wm)
{
	struct intel_crtc *crtc;
1517
	int num_active_pipes = 0;
1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535

	wm->cxsr = true;
	wm->hpll_en = true;
	wm->fbc_en = true;

	for_each_intel_crtc(&dev_priv->drm, crtc) {
		const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;

		if (!crtc->active)
			continue;

		if (!wm_state->cxsr)
			wm->cxsr = false;
		if (!wm_state->hpll_en)
			wm->hpll_en = false;
		if (!wm_state->fbc_en)
			wm->fbc_en = false;

1536
		num_active_pipes++;
1537 1538
	}

1539
	if (num_active_pipes != 1) {
1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578
		wm->cxsr = false;
		wm->hpll_en = false;
		wm->fbc_en = false;
	}

	for_each_intel_crtc(&dev_priv->drm, crtc) {
		const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
		enum pipe pipe = crtc->pipe;

		wm->pipe[pipe] = wm_state->wm;
		if (crtc->active && wm->cxsr)
			wm->sr = wm_state->sr;
		if (crtc->active && wm->hpll_en)
			wm->hpll = wm_state->hpll;
	}
}

static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
{
	struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
	struct g4x_wm_values new_wm = {};

	g4x_merge_wm(dev_priv, &new_wm);

	if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
		return;

	if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
		_intel_set_memory_cxsr(dev_priv, false);

	g4x_write_wm_values(dev_priv, &new_wm);

	if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
		_intel_set_memory_cxsr(dev_priv, true);

	*old_wm = new_wm;
}

static void g4x_initial_watermarks(struct intel_atomic_state *state,
1579
				   struct intel_crtc *crtc)
1580
{
1581 1582 1583
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	const struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
1584 1585 1586 1587 1588 1589 1590 1591

	mutex_lock(&dev_priv->wm.wm_mutex);
	crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
	g4x_program_watermarks(dev_priv);
	mutex_unlock(&dev_priv->wm.wm_mutex);
}

static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1592
				    struct intel_crtc *crtc)
1593
{
1594 1595 1596
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	const struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
1597 1598 1599 1600 1601

	if (!crtc_state->wm.need_postvbl_update)
		return;

	mutex_lock(&dev_priv->wm.wm_mutex);
1602
	crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1603 1604 1605 1606
	g4x_program_watermarks(dev_priv);
	mutex_unlock(&dev_priv->wm.wm_mutex);
}

1607 1608
/* latency must be in 0.1us units. */
static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1609 1610
				   unsigned int htotal,
				   unsigned int width,
1611
				   unsigned int cpp,
1612 1613 1614 1615
				   unsigned int latency)
{
	unsigned int ret;

1616 1617
	ret = intel_wm_method2(pixel_rate, htotal,
			       width, cpp, latency);
1618 1619 1620 1621 1622
	ret = DIV_ROUND_UP(ret, 64);

	return ret;
}

1623
static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1624 1625 1626 1627
{
	/* all latencies in usec */
	dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;

1628 1629
	dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;

1630 1631 1632
	if (IS_CHERRYVIEW(dev_priv)) {
		dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
		dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1633 1634

		dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1635 1636 1637
	}
}

1638 1639 1640
static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
				const struct intel_plane_state *plane_state,
				int level)
1641
{
1642
	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1643
	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1644
	const struct drm_display_mode *adjusted_mode =
1645
		&crtc_state->hw.adjusted_mode;
1646
	unsigned int clock, htotal, cpp, width, wm;
1647 1648 1649 1650

	if (dev_priv->wm.pri_latency[level] == 0)
		return USHRT_MAX;

1651
	if (!intel_wm_plane_visible(crtc_state, plane_state))
1652 1653
		return 0;

1654
	cpp = plane_state->hw.fb->format->cpp[0];
1655 1656 1657
	clock = adjusted_mode->crtc_clock;
	htotal = adjusted_mode->crtc_htotal;
	width = crtc_state->pipe_src_w;
1658

1659
	if (plane->id == PLANE_CURSOR) {
1660 1661 1662 1663 1664 1665 1666 1667
		/*
		 * FIXME the formula gives values that are
		 * too big for the cursor FIFO, and hence we
		 * would never be able to use cursors. For
		 * now just hardcode the watermark.
		 */
		wm = 63;
	} else {
1668
		wm = vlv_wm_method2(clock, htotal, width, cpp,
1669 1670 1671
				    dev_priv->wm.pri_latency[level] * 10);
	}

1672
	return min_t(unsigned int, wm, USHRT_MAX);
1673 1674
}

1675 1676 1677 1678 1679 1680
static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
{
	return (active_planes & (BIT(PLANE_SPRITE0) |
				 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
}

1681
static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1682
{
1683
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1684
	const struct g4x_pipe_wm *raw =
1685
		&crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1686
	struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1687
	unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1688
	int num_active_planes = hweight8(active_planes);
1689
	const int fifo_size = 511;
1690
	int fifo_extra, fifo_left = fifo_size;
1691
	int sprite0_fifo_extra = 0;
1692 1693
	unsigned int total_rate;
	enum plane_id plane_id;
1694

1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705
	/*
	 * When enabling sprite0 after sprite1 has already been enabled
	 * we tend to get an underrun unless sprite0 already has some
	 * FIFO space allcoated. Hence we always allocate at least one
	 * cacheline for sprite0 whenever sprite1 is enabled.
	 *
	 * All other plane enable sequences appear immune to this problem.
	 */
	if (vlv_need_sprite0_fifo_workaround(active_planes))
		sprite0_fifo_extra = 1;

1706 1707
	total_rate = raw->plane[PLANE_PRIMARY] +
		raw->plane[PLANE_SPRITE0] +
1708 1709
		raw->plane[PLANE_SPRITE1] +
		sprite0_fifo_extra;
1710

1711 1712
	if (total_rate > fifo_size)
		return -EINVAL;
1713

1714 1715
	if (total_rate == 0)
		total_rate = 1;
1716

1717
	for_each_plane_id_on_crtc(crtc, plane_id) {
1718 1719
		unsigned int rate;

1720 1721
		if ((active_planes & BIT(plane_id)) == 0) {
			fifo_state->plane[plane_id] = 0;
1722 1723 1724
			continue;
		}

1725 1726 1727
		rate = raw->plane[plane_id];
		fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
		fifo_left -= fifo_state->plane[plane_id];
1728 1729
	}

1730 1731 1732
	fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
	fifo_left -= sprite0_fifo_extra;

1733 1734 1735
	fifo_state->plane[PLANE_CURSOR] = 63;

	fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1736 1737

	/* spread the remainder evenly */
1738
	for_each_plane_id_on_crtc(crtc, plane_id) {
1739 1740 1741 1742 1743
		int plane_extra;

		if (fifo_left == 0)
			break;

1744
		if ((active_planes & BIT(plane_id)) == 0)
1745 1746 1747
			continue;

		plane_extra = min(fifo_extra, fifo_left);
1748
		fifo_state->plane[plane_id] += plane_extra;
1749 1750 1751
		fifo_left -= plane_extra;
	}

1752 1753 1754 1755 1756 1757 1758 1759 1760
	WARN_ON(active_planes != 0 && fifo_left != 0);

	/* give it all to the first plane if none are active */
	if (active_planes == 0) {
		WARN_ON(fifo_left != fifo_size);
		fifo_state->plane[PLANE_PRIMARY] = fifo_left;
	}

	return 0;
1761 1762
}

1763 1764 1765 1766 1767 1768
/* mark all levels starting from 'level' as invalid */
static void vlv_invalidate_wms(struct intel_crtc *crtc,
			       struct vlv_wm_state *wm_state, int level)
{
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);

1769
	for (; level < intel_wm_num_levels(dev_priv); level++) {
1770 1771 1772 1773 1774 1775 1776 1777 1778 1779
		enum plane_id plane_id;

		for_each_plane_id_on_crtc(crtc, plane_id)
			wm_state->wm[level].plane[plane_id] = USHRT_MAX;

		wm_state->sr[level].cursor = USHRT_MAX;
		wm_state->sr[level].plane = USHRT_MAX;
	}
}

1780 1781 1782 1783 1784 1785 1786 1787
static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
{
	if (wm > fifo_size)
		return USHRT_MAX;
	else
		return fifo_size - wm;
}

1788 1789 1790 1791
/*
 * Starting from 'level' set all higher
 * levels to 'value' in the "raw" watermarks.
 */
1792
static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1793
				 int level, enum plane_id plane_id, u16 value)
1794
{
1795
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1796
	int num_levels = intel_wm_num_levels(dev_priv);
1797
	bool dirty = false;
1798

1799
	for (; level < num_levels; level++) {
1800
		struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1801

1802
		dirty |= raw->plane[plane_id] != value;
1803
		raw->plane[plane_id] = value;
1804
	}
1805 1806

	return dirty;
1807 1808
}

1809 1810
static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
				     const struct intel_plane_state *plane_state)
1811
{
1812
	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1813
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1814
	enum plane_id plane_id = plane->id;
1815
	int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1816
	int level;
1817
	bool dirty = false;
1818

1819
	if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1820 1821
		dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
		goto out;
1822
	}
1823

1824
	for (level = 0; level < num_levels; level++) {
1825
		struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1826 1827
		int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
		int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1828

1829 1830
		if (wm > max_wm)
			break;
1831

1832
		dirty |= raw->plane[plane_id] != wm;
1833 1834
		raw->plane[plane_id] = wm;
	}
1835

1836
	/* mark all higher levels as invalid */
1837
	dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1838

1839 1840
out:
	if (dirty)
1841 1842 1843 1844 1845 1846
		drm_dbg_kms(&dev_priv->drm,
			    "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
			    plane->base.name,
			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1847 1848

	return dirty;
1849
}
1850

1851 1852
static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
				      enum plane_id plane_id, int level)
1853
{
1854
	const struct g4x_pipe_wm *raw =
1855 1856 1857
		&crtc_state->wm.vlv.raw[level];
	const struct vlv_fifo_state *fifo_state =
		&crtc_state->wm.vlv.fifo_state;
1858

1859 1860
	return raw->plane[plane_id] <= fifo_state->plane[plane_id];
}
1861

1862
static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1863
{
1864 1865 1866 1867
	return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1868 1869 1870 1871
}

static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
{
1872
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1873 1874
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	struct intel_atomic_state *state =
1875
		to_intel_atomic_state(crtc_state->uapi.state);
1876 1877 1878
	struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
	const struct vlv_fifo_state *fifo_state =
		&crtc_state->wm.vlv.fifo_state;
1879 1880
	int num_active_planes = hweight8(crtc_state->active_planes &
					 ~BIT(PLANE_CURSOR));
1881
	bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1882 1883
	const struct intel_plane_state *old_plane_state;
	const struct intel_plane_state *new_plane_state;
1884 1885 1886
	struct intel_plane *plane;
	enum plane_id plane_id;
	int level, ret, i;
1887
	unsigned int dirty = 0;
1888

1889 1890 1891
	for_each_oldnew_intel_plane_in_state(state, plane,
					     old_plane_state,
					     new_plane_state, i) {
1892 1893
		if (new_plane_state->hw.crtc != &crtc->base &&
		    old_plane_state->hw.crtc != &crtc->base)
1894
			continue;
1895

1896
		if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914
			dirty |= BIT(plane->id);
	}

	/*
	 * DSPARB registers may have been reset due to the
	 * power well being turned off. Make sure we restore
	 * them to a consistent state even if no primary/sprite
	 * planes are initially active.
	 */
	if (needs_modeset)
		crtc_state->fifo_changed = true;

	if (!dirty)
		return 0;

	/* cursor changes don't warrant a FIFO recompute */
	if (dirty & ~BIT(PLANE_CURSOR)) {
		const struct intel_crtc_state *old_crtc_state =
1915
			intel_atomic_get_old_crtc_state(state, crtc);
1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926
		const struct vlv_fifo_state *old_fifo_state =
			&old_crtc_state->wm.vlv.fifo_state;

		ret = vlv_compute_fifo(crtc_state);
		if (ret)
			return ret;

		if (needs_modeset ||
		    memcmp(old_fifo_state, fifo_state,
			   sizeof(*fifo_state)) != 0)
			crtc_state->fifo_changed = true;
1927
	}
1928

1929
	/* initially allow all levels */
1930
	wm_state->num_levels = intel_wm_num_levels(dev_priv);
1931 1932 1933 1934 1935
	/*
	 * Note that enabling cxsr with no primary/sprite planes
	 * enabled can wedge the pipe. Hence we only allow cxsr
	 * with exactly one enabled primary/sprite plane.
	 */
1936
	wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1937

1938
	for (level = 0; level < wm_state->num_levels; level++) {
1939
		const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1940
		const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1941

1942
		if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1943
			break;
1944

1945 1946 1947 1948 1949 1950 1951 1952
		for_each_plane_id_on_crtc(crtc, plane_id) {
			wm_state->wm[level].plane[plane_id] =
				vlv_invert_wm_value(raw->plane[plane_id],
						    fifo_state->plane[plane_id]);
		}

		wm_state->sr[level].plane =
			vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1953
						 raw->plane[PLANE_SPRITE0],
1954 1955
						 raw->plane[PLANE_SPRITE1]),
					    sr_fifo_size);
1956

1957 1958 1959
		wm_state->sr[level].cursor =
			vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
					    63);
1960 1961
	}

1962 1963 1964 1965 1966 1967 1968 1969 1970 1971
	if (level == 0)
		return -EINVAL;

	/* limit to only levels we can actually handle */
	wm_state->num_levels = level;

	/* invalidate the higher levels */
	vlv_invalidate_wms(crtc, wm_state, level);

	return 0;
1972 1973
}

1974 1975 1976
#define VLV_FIFO(plane, value) \
	(((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)

1977
static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1978
				   struct intel_crtc *crtc)
1979
{
1980
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1981
	struct intel_uncore *uncore = &dev_priv->uncore;
1982 1983
	const struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
1984 1985
	const struct vlv_fifo_state *fifo_state =
		&crtc_state->wm.vlv.fifo_state;
1986
	int sprite0_start, sprite1_start, fifo_size;
1987
	u32 dsparb, dsparb2, dsparb3;
1988

1989 1990 1991
	if (!crtc_state->fifo_changed)
		return;

1992 1993 1994
	sprite0_start = fifo_state->plane[PLANE_PRIMARY];
	sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
	fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1995

1996 1997
	drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
	drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
1998

1999 2000
	trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);

2001 2002 2003 2004 2005 2006 2007 2008 2009
	/*
	 * uncore.lock serves a double purpose here. It allows us to
	 * use the less expensive I915_{READ,WRITE}_FW() functions, and
	 * it protects the DSPARB registers from getting clobbered by
	 * parallel updates from multiple pipes.
	 *
	 * intel_pipe_update_start() has already disabled interrupts
	 * for us, so a plain spin_lock() is sufficient here.
	 */
2010
	spin_lock(&uncore->lock);
2011

2012 2013
	switch (crtc->pipe) {
	case PIPE_A:
2014 2015
		dsparb = intel_uncore_read_fw(uncore, DSPARB);
		dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026

		dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
			    VLV_FIFO(SPRITEB, 0xff));
		dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
			   VLV_FIFO(SPRITEB, sprite1_start));

		dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
			     VLV_FIFO(SPRITEB_HI, 0x1));
		dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
			   VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));

2027 2028
		intel_uncore_write_fw(uncore, DSPARB, dsparb);
		intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2029 2030
		break;
	case PIPE_B:
2031 2032
		dsparb = intel_uncore_read_fw(uncore, DSPARB);
		dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043

		dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
			    VLV_FIFO(SPRITED, 0xff));
		dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
			   VLV_FIFO(SPRITED, sprite1_start));

		dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
			     VLV_FIFO(SPRITED_HI, 0xff));
		dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
			   VLV_FIFO(SPRITED_HI, sprite1_start >> 8));

2044 2045
		intel_uncore_write_fw(uncore, DSPARB, dsparb);
		intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2046 2047
		break;
	case PIPE_C:
2048 2049
		dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
		dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060

		dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
			     VLV_FIFO(SPRITEF, 0xff));
		dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
			    VLV_FIFO(SPRITEF, sprite1_start));

		dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
			     VLV_FIFO(SPRITEF_HI, 0xff));
		dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
			   VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));

2061 2062
		intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
		intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2063 2064 2065 2066
		break;
	default:
		break;
	}
2067

2068
	intel_uncore_posting_read_fw(uncore, DSPARB);
2069

2070
	spin_unlock(&uncore->lock);
2071 2072 2073 2074
}

#undef VLV_FIFO

2075
static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
2076
{
2077
	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2078 2079 2080
	struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
	const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
	struct intel_atomic_state *intel_state =
2081
		to_intel_atomic_state(new_crtc_state->uapi.state);
2082 2083 2084
	const struct intel_crtc_state *old_crtc_state =
		intel_atomic_get_old_crtc_state(intel_state, crtc);
	const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2085 2086
	int level;

2087
	if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2088 2089 2090 2091 2092 2093
		*intermediate = *optimal;

		intermediate->cxsr = false;
		goto out;
	}

2094
	intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2095
	intermediate->cxsr = optimal->cxsr && active->cxsr &&
2096
		!new_crtc_state->disable_cxsr;
2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114

	for (level = 0; level < intermediate->num_levels; level++) {
		enum plane_id plane_id;

		for_each_plane_id_on_crtc(crtc, plane_id) {
			intermediate->wm[level].plane[plane_id] =
				min(optimal->wm[level].plane[plane_id],
				    active->wm[level].plane[plane_id]);
		}

		intermediate->sr[level].plane = min(optimal->sr[level].plane,
						    active->sr[level].plane);
		intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
						     active->sr[level].cursor);
	}

	vlv_invalidate_wms(crtc, intermediate, level);

2115
out:
2116 2117 2118 2119
	/*
	 * If our intermediate WM are identical to the final WM, then we can
	 * omit the post-vblank programming; only update if it's different.
	 */
2120
	if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2121
		new_crtc_state->wm.need_postvbl_update = true;
2122 2123 2124 2125

	return 0;
}

2126
static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2127 2128 2129
			 struct vlv_wm_values *wm)
{
	struct intel_crtc *crtc;
2130
	int num_active_pipes = 0;
2131

2132
	wm->level = dev_priv->wm.max_level;
2133 2134
	wm->cxsr = true;

2135
	for_each_intel_crtc(&dev_priv->drm, crtc) {
2136
		const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2137 2138 2139 2140 2141 2142 2143

		if (!crtc->active)
			continue;

		if (!wm_state->cxsr)
			wm->cxsr = false;

2144
		num_active_pipes++;
2145 2146 2147
		wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
	}

2148
	if (num_active_pipes != 1)
2149 2150
		wm->cxsr = false;

2151
	if (num_active_pipes > 1)
2152 2153
		wm->level = VLV_WM_LEVEL_PM2;

2154
	for_each_intel_crtc(&dev_priv->drm, crtc) {
2155
		const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2156 2157 2158
		enum pipe pipe = crtc->pipe;

		wm->pipe[pipe] = wm_state->wm[wm->level];
2159
		if (crtc->active && wm->cxsr)
2160 2161
			wm->sr = wm_state->sr[wm->level];

2162 2163 2164 2165
		wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
		wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
		wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
		wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2166 2167 2168
	}
}

2169
static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2170
{
2171 2172
	struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
	struct vlv_wm_values new_wm = {};
2173

2174
	vlv_merge_wm(dev_priv, &new_wm);
2175

2176
	if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2177 2178
		return;

2179
	if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2180 2181
		chv_set_memory_dvfs(dev_priv, false);

2182
	if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2183 2184
		chv_set_memory_pm5(dev_priv, false);

2185
	if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2186
		_intel_set_memory_cxsr(dev_priv, false);
2187

2188
	vlv_write_wm_values(dev_priv, &new_wm);
2189

2190
	if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2191
		_intel_set_memory_cxsr(dev_priv, true);
2192

2193
	if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2194 2195
		chv_set_memory_pm5(dev_priv, true);

2196
	if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2197 2198
		chv_set_memory_dvfs(dev_priv, true);

2199
	*old_wm = new_wm;
2200 2201
}

2202
static void vlv_initial_watermarks(struct intel_atomic_state *state,
2203
				   struct intel_crtc *crtc)
2204
{
2205 2206 2207
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	const struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
2208 2209

	mutex_lock(&dev_priv->wm.wm_mutex);
2210 2211 2212 2213 2214 2215
	crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
	vlv_program_watermarks(dev_priv);
	mutex_unlock(&dev_priv->wm.wm_mutex);
}

static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2216
				    struct intel_crtc *crtc)
2217
{
2218 2219 2220
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	const struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
2221 2222 2223 2224 2225

	if (!crtc_state->wm.need_postvbl_update)
		return;

	mutex_lock(&dev_priv->wm.wm_mutex);
2226
	crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2227 2228 2229 2230
	vlv_program_watermarks(dev_priv);
	mutex_unlock(&dev_priv->wm.wm_mutex);
}

2231
static void i965_update_wm(struct intel_crtc *unused_crtc)
2232
{
2233
	struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2234
	struct intel_crtc *crtc;
2235 2236
	int srwm = 1;
	int cursor_sr = 16;
2237
	bool cxsr_enabled;
2238 2239

	/* Calc sr entries for one plane configs */
2240
	crtc = single_enabled_crtc(dev_priv);
2241 2242 2243
	if (crtc) {
		/* self-refresh has much higher latency */
		static const int sr_latency_ns = 12000;
2244
		const struct drm_display_mode *adjusted_mode =
2245
			&crtc->config->hw.adjusted_mode;
2246 2247
		const struct drm_framebuffer *fb =
			crtc->base.primary->state->fb;
2248
		int clock = adjusted_mode->crtc_clock;
2249
		int htotal = adjusted_mode->crtc_htotal;
2250
		int hdisplay = crtc->config->pipe_src_w;
2251
		int cpp = fb->format->cpp[0];
2252 2253
		int entries;

2254 2255
		entries = intel_wm_method2(clock, htotal,
					   hdisplay, cpp, sr_latency_ns / 100);
2256 2257 2258 2259 2260
		entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		srwm = I965_FIFO_SIZE - entries;
		if (srwm < 0)
			srwm = 1;
		srwm &= 0x1ff;
2261 2262 2263
		drm_dbg_kms(&dev_priv->drm,
			    "self-refresh entries: %d, wm: %d\n",
			    entries, srwm);
2264

2265 2266 2267
		entries = intel_wm_method2(clock, htotal,
					   crtc->base.cursor->state->crtc_w, 4,
					   sr_latency_ns / 100);
2268
		entries = DIV_ROUND_UP(entries,
2269 2270
				       i965_cursor_wm_info.cacheline_size) +
			i965_cursor_wm_info.guard_size;
2271

2272
		cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2273 2274 2275
		if (cursor_sr > i965_cursor_wm_info.max_wm)
			cursor_sr = i965_cursor_wm_info.max_wm;

2276 2277 2278
		drm_dbg_kms(&dev_priv->drm,
			    "self-refresh watermark: display plane %d "
			    "cursor %d\n", srwm, cursor_sr);
2279

2280
		cxsr_enabled = true;
2281
	} else {
2282
		cxsr_enabled = false;
2283
		/* Turn off self refresh if both pipes are enabled */
2284
		intel_set_memory_cxsr(dev_priv, false);
2285 2286
	}

2287 2288 2289
	drm_dbg_kms(&dev_priv->drm,
		    "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		    srwm);
2290 2291

	/* 965 has limitations... */
2292 2293 2294 2295 2296 2297
	I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
		   FW_WM(8, CURSORB) |
		   FW_WM(8, PLANEB) |
		   FW_WM(8, PLANEA));
	I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
		   FW_WM(8, PLANEC_OLD));
2298
	/* update cursor SR watermark */
2299
	I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2300 2301 2302

	if (cxsr_enabled)
		intel_set_memory_cxsr(dev_priv, true);
2303 2304
}

2305 2306
#undef FW_WM

2307
static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2308
{
2309
	struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2310
	const struct intel_watermark_params *wm_info;
2311 2312
	u32 fwater_lo;
	u32 fwater_hi;
2313 2314 2315
	int cwm, srwm = 1;
	int fifo_size;
	int planea_wm, planeb_wm;
2316
	struct intel_crtc *crtc, *enabled = NULL;
2317

2318
	if (IS_I945GM(dev_priv))
2319
		wm_info = &i945_wm_info;
2320
	else if (!IS_GEN(dev_priv, 2))
2321 2322
		wm_info = &i915_wm_info;
	else
2323
		wm_info = &i830_a_wm_info;
2324

2325 2326
	fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
	crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2327 2328
	if (intel_crtc_active(crtc)) {
		const struct drm_display_mode *adjusted_mode =
2329
			&crtc->config->hw.adjusted_mode;
2330 2331 2332 2333
		const struct drm_framebuffer *fb =
			crtc->base.primary->state->fb;
		int cpp;

2334
		if (IS_GEN(dev_priv, 2))
2335
			cpp = 4;
2336
		else
2337
			cpp = fb->format->cpp[0];
2338

2339
		planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2340
					       wm_info, fifo_size, cpp,
2341
					       pessimal_latency_ns);
2342
		enabled = crtc;
2343
	} else {
2344
		planea_wm = fifo_size - wm_info->guard_size;
2345 2346 2347 2348
		if (planea_wm > (long)wm_info->max_wm)
			planea_wm = wm_info->max_wm;
	}

2349
	if (IS_GEN(dev_priv, 2))
2350
		wm_info = &i830_bc_wm_info;
2351

2352 2353
	fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
	crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2354 2355
	if (intel_crtc_active(crtc)) {
		const struct drm_display_mode *adjusted_mode =
2356
			&crtc->config->hw.adjusted_mode;
2357 2358 2359 2360
		const struct drm_framebuffer *fb =
			crtc->base.primary->state->fb;
		int cpp;

2361
		if (IS_GEN(dev_priv, 2))
2362
			cpp = 4;
2363
		else
2364
			cpp = fb->format->cpp[0];
2365

2366
		planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2367
					       wm_info, fifo_size, cpp,
2368
					       pessimal_latency_ns);
2369 2370 2371 2372
		if (enabled == NULL)
			enabled = crtc;
		else
			enabled = NULL;
2373
	} else {
2374
		planeb_wm = fifo_size - wm_info->guard_size;
2375 2376 2377
		if (planeb_wm > (long)wm_info->max_wm)
			planeb_wm = wm_info->max_wm;
	}
2378

2379 2380
	drm_dbg_kms(&dev_priv->drm,
		    "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2381

2382
	if (IS_I915GM(dev_priv) && enabled) {
2383
		struct drm_i915_gem_object *obj;
2384

2385
		obj = intel_fb_obj(enabled->base.primary->state->fb);
2386 2387

		/* self-refresh seems busted with untiled */
2388
		if (!i915_gem_object_is_tiled(obj))
2389 2390 2391
			enabled = NULL;
	}

2392 2393 2394 2395 2396 2397
	/*
	 * Overlay gets an aggressive default since video jitter is bad.
	 */
	cwm = 2;

	/* Play safe and disable self-refresh before adjusting watermarks. */
2398
	intel_set_memory_cxsr(dev_priv, false);
2399 2400

	/* Calc sr entries for one plane configs */
2401
	if (HAS_FW_BLC(dev_priv) && enabled) {
2402 2403
		/* self-refresh has much higher latency */
		static const int sr_latency_ns = 6000;
2404
		const struct drm_display_mode *adjusted_mode =
2405
			&enabled->config->hw.adjusted_mode;
2406 2407
		const struct drm_framebuffer *fb =
			enabled->base.primary->state->fb;
2408
		int clock = adjusted_mode->crtc_clock;
2409
		int htotal = adjusted_mode->crtc_htotal;
2410 2411
		int hdisplay = enabled->config->pipe_src_w;
		int cpp;
2412 2413
		int entries;

2414
		if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2415
			cpp = 4;
2416
		else
2417
			cpp = fb->format->cpp[0];
2418

2419 2420
		entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
					   sr_latency_ns / 100);
2421
		entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2422 2423
		drm_dbg_kms(&dev_priv->drm,
			    "self-refresh entries: %d\n", entries);
2424 2425 2426 2427
		srwm = wm_info->fifo_size - entries;
		if (srwm < 0)
			srwm = 1;

2428
		if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2429 2430
			I915_WRITE(FW_BLC_SELF,
				   FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2431
		else
2432 2433 2434
			I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
	}

2435 2436 2437
	drm_dbg_kms(&dev_priv->drm,
		    "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		     planea_wm, planeb_wm, cwm, srwm);
2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448

	fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
	fwater_hi = (cwm & 0x1f);

	/* Set request length to 8 cachelines per fetch */
	fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
	fwater_hi = fwater_hi | (1 << 8);

	I915_WRITE(FW_BLC, fwater_lo);
	I915_WRITE(FW_BLC2, fwater_hi);

2449 2450
	if (enabled)
		intel_set_memory_cxsr(dev_priv, true);
2451 2452
}

2453
static void i845_update_wm(struct intel_crtc *unused_crtc)
2454
{
2455
	struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2456
	struct intel_crtc *crtc;
2457
	const struct drm_display_mode *adjusted_mode;
2458
	u32 fwater_lo;
2459 2460
	int planea_wm;

2461
	crtc = single_enabled_crtc(dev_priv);
2462 2463 2464
	if (crtc == NULL)
		return;

2465
	adjusted_mode = &crtc->config->hw.adjusted_mode;
2466
	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2467
				       &i845_wm_info,
2468
				       dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2469
				       4, pessimal_latency_ns);
2470 2471 2472
	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
	fwater_lo |= (3<<8) | planea_wm;

2473 2474
	drm_dbg_kms(&dev_priv->drm,
		    "Setting FIFO watermarks - A: %d\n", planea_wm);
2475 2476 2477 2478

	I915_WRITE(FW_BLC, fwater_lo);
}

2479
/* latency must be in 0.1us units. */
2480 2481 2482
static unsigned int ilk_wm_method1(unsigned int pixel_rate,
				   unsigned int cpp,
				   unsigned int latency)
2483
{
2484
	unsigned int ret;
2485

2486 2487
	ret = intel_wm_method1(pixel_rate, cpp, latency);
	ret = DIV_ROUND_UP(ret, 64) + 2;
2488 2489 2490 2491

	return ret;
}

2492
/* latency must be in 0.1us units. */
2493 2494 2495 2496 2497
static unsigned int ilk_wm_method2(unsigned int pixel_rate,
				   unsigned int htotal,
				   unsigned int width,
				   unsigned int cpp,
				   unsigned int latency)
2498
{
2499
	unsigned int ret;
2500

2501 2502
	ret = intel_wm_method2(pixel_rate, htotal,
			       width, cpp, latency);
2503
	ret = DIV_ROUND_UP(ret, 64) + 2;
2504

2505 2506 2507
	return ret;
}

2508
static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2509
{
2510 2511 2512 2513 2514 2515
	/*
	 * Neither of these should be possible since this function shouldn't be
	 * called if the CRTC is off or the plane is invisible.  But let's be
	 * extra paranoid to avoid a potential divide-by-zero if we screw up
	 * elsewhere in the driver.
	 */
2516
	if (WARN_ON(!cpp))
2517 2518 2519 2520
		return 0;
	if (WARN_ON(!horiz_pixels))
		return 0;

2521
	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2522 2523
}

2524
struct ilk_wm_maximums {
2525 2526 2527 2528
	u16 pri;
	u16 spr;
	u16 cur;
	u16 fbc;
2529 2530
};

2531 2532 2533 2534
/*
 * For both WM_PIPE and WM_LP.
 * mem_value must be in 0.1us units.
 */
2535 2536
static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
			      const struct intel_plane_state *plane_state,
2537
			      u32 mem_value, bool is_lp)
2538
{
2539
	u32 method1, method2;
2540
	int cpp;
2541

2542 2543 2544
	if (mem_value == 0)
		return U32_MAX;

2545
	if (!intel_wm_plane_visible(crtc_state, plane_state))
2546 2547
		return 0;

2548
	cpp = plane_state->hw.fb->format->cpp[0];
2549

2550
	method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2551 2552 2553 2554

	if (!is_lp)
		return method1;

2555
	method2 = ilk_wm_method2(crtc_state->pixel_rate,
2556
				 crtc_state->hw.adjusted_mode.crtc_htotal,
2557
				 drm_rect_width(&plane_state->uapi.dst),
2558
				 cpp, mem_value);
2559 2560

	return min(method1, method2);
2561 2562
}

2563 2564 2565 2566
/*
 * For both WM_PIPE and WM_LP.
 * mem_value must be in 0.1us units.
 */
2567 2568
static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
			      const struct intel_plane_state *plane_state,
2569
			      u32 mem_value)
2570
{
2571
	u32 method1, method2;
2572
	int cpp;
2573

2574 2575 2576
	if (mem_value == 0)
		return U32_MAX;

2577
	if (!intel_wm_plane_visible(crtc_state, plane_state))
2578 2579
		return 0;

2580
	cpp = plane_state->hw.fb->format->cpp[0];
2581

2582 2583
	method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
	method2 = ilk_wm_method2(crtc_state->pixel_rate,
2584
				 crtc_state->hw.adjusted_mode.crtc_htotal,
2585
				 drm_rect_width(&plane_state->uapi.dst),
2586
				 cpp, mem_value);
2587 2588 2589
	return min(method1, method2);
}

2590 2591 2592 2593
/*
 * For both WM_PIPE and WM_LP.
 * mem_value must be in 0.1us units.
 */
2594 2595
static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
			      const struct intel_plane_state *plane_state,
2596
			      u32 mem_value)
2597
{
2598 2599
	int cpp;

2600 2601 2602
	if (mem_value == 0)
		return U32_MAX;

2603
	if (!intel_wm_plane_visible(crtc_state, plane_state))
2604 2605
		return 0;

2606
	cpp = plane_state->hw.fb->format->cpp[0];
2607

2608
	return ilk_wm_method2(crtc_state->pixel_rate,
2609
			      crtc_state->hw.adjusted_mode.crtc_htotal,
2610
			      drm_rect_width(&plane_state->uapi.dst),
2611
			      cpp, mem_value);
2612 2613
}

2614
/* Only for WM_LP. */
2615 2616
static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
			      const struct intel_plane_state *plane_state,
2617
			      u32 pri_val)
2618
{
2619
	int cpp;
2620

2621
	if (!intel_wm_plane_visible(crtc_state, plane_state))
2622 2623
		return 0;

2624
	cpp = plane_state->hw.fb->format->cpp[0];
2625

2626 2627
	return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
			  cpp);
2628 2629
}

2630 2631
static unsigned int
ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2632
{
2633
	if (INTEL_GEN(dev_priv) >= 8)
2634
		return 3072;
2635
	else if (INTEL_GEN(dev_priv) >= 7)
2636 2637 2638 2639 2640
		return 768;
	else
		return 512;
}

2641 2642 2643
static unsigned int
ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
		     int level, bool is_sprite)
2644
{
2645
	if (INTEL_GEN(dev_priv) >= 8)
2646 2647
		/* BDW primary/sprite plane watermarks */
		return level == 0 ? 255 : 2047;
2648
	else if (INTEL_GEN(dev_priv) >= 7)
2649 2650 2651 2652 2653 2654 2655 2656 2657 2658
		/* IVB/HSW primary/sprite plane watermarks */
		return level == 0 ? 127 : 1023;
	else if (!is_sprite)
		/* ILK/SNB primary plane watermarks */
		return level == 0 ? 127 : 511;
	else
		/* ILK/SNB sprite plane watermarks */
		return level == 0 ? 63 : 255;
}

2659 2660
static unsigned int
ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2661
{
2662
	if (INTEL_GEN(dev_priv) >= 7)
2663 2664 2665 2666 2667
		return level == 0 ? 63 : 255;
	else
		return level == 0 ? 31 : 63;
}

2668
static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2669
{
2670
	if (INTEL_GEN(dev_priv) >= 8)
2671 2672 2673 2674 2675
		return 31;
	else
		return 15;
}

2676
/* Calculate the maximum primary/sprite plane watermark */
2677
static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2678
				     int level,
2679
				     const struct intel_wm_config *config,
2680 2681 2682
				     enum intel_ddb_partitioning ddb_partitioning,
				     bool is_sprite)
{
2683
	unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2684 2685

	/* if sprites aren't enabled, sprites get nothing */
2686
	if (is_sprite && !config->sprites_enabled)
2687 2688 2689
		return 0;

	/* HSW allows LP1+ watermarks even with multiple pipes */
2690
	if (level == 0 || config->num_pipes_active > 1) {
2691
		fifo_size /= INTEL_NUM_PIPES(dev_priv);
2692 2693 2694 2695 2696 2697

		/*
		 * For some reason the non self refresh
		 * FIFO size is only half of the self
		 * refresh FIFO size on ILK/SNB.
		 */
2698
		if (INTEL_GEN(dev_priv) <= 6)
2699 2700 2701
			fifo_size /= 2;
	}

2702
	if (config->sprites_enabled) {
2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713
		/* level 0 is always calculated with 1:1 split */
		if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
			if (is_sprite)
				fifo_size *= 5;
			fifo_size /= 6;
		} else {
			fifo_size /= 2;
		}
	}

	/* clamp to max that the registers can hold */
2714
	return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2715 2716 2717
}

/* Calculate the maximum cursor plane watermark */
2718
static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2719 2720
				      int level,
				      const struct intel_wm_config *config)
2721 2722
{
	/* HSW LP1+ watermarks w/ multiple pipes */
2723
	if (level > 0 && config->num_pipes_active > 1)
2724 2725 2726
		return 64;

	/* otherwise just report max that registers can hold */
2727
	return ilk_cursor_wm_reg_max(dev_priv, level);
2728 2729
}

2730
static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2731 2732 2733
				    int level,
				    const struct intel_wm_config *config,
				    enum intel_ddb_partitioning ddb_partitioning,
2734
				    struct ilk_wm_maximums *max)
2735
{
2736 2737 2738 2739
	max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
	max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
	max->cur = ilk_cursor_wm_max(dev_priv, level, config);
	max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2740 2741
}

2742
static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2743 2744 2745
					int level,
					struct ilk_wm_maximums *max)
{
2746 2747 2748 2749
	max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
	max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
	max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
	max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2750 2751
}

2752
static bool ilk_validate_wm_level(int level,
2753
				  const struct ilk_wm_maximums *max,
2754
				  struct intel_wm_level *result)
2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783
{
	bool ret;

	/* already determined to be invalid? */
	if (!result->enable)
		return false;

	result->enable = result->pri_val <= max->pri &&
			 result->spr_val <= max->spr &&
			 result->cur_val <= max->cur;

	ret = result->enable;

	/*
	 * HACK until we can pre-compute everything,
	 * and thus fail gracefully if LP0 watermarks
	 * are exceeded...
	 */
	if (level == 0 && !result->enable) {
		if (result->pri_val > max->pri)
			DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
				      level, result->pri_val, max->pri);
		if (result->spr_val > max->spr)
			DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
				      level, result->spr_val, max->spr);
		if (result->cur_val > max->cur)
			DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
				      level, result->cur_val, max->cur);

2784 2785 2786
		result->pri_val = min_t(u32, result->pri_val, max->pri);
		result->spr_val = min_t(u32, result->spr_val, max->spr);
		result->cur_val = min_t(u32, result->cur_val, max->cur);
2787 2788 2789 2790 2791 2792
		result->enable = true;
	}

	return ret;
}

2793
static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2794
				 const struct intel_crtc *crtc,
2795
				 int level,
2796
				 struct intel_crtc_state *crtc_state,
2797 2798 2799
				 const struct intel_plane_state *pristate,
				 const struct intel_plane_state *sprstate,
				 const struct intel_plane_state *curstate,
2800
				 struct intel_wm_level *result)
2801
{
2802 2803 2804
	u16 pri_latency = dev_priv->wm.pri_latency[level];
	u16 spr_latency = dev_priv->wm.spr_latency[level];
	u16 cur_latency = dev_priv->wm.cur_latency[level];
2805 2806 2807 2808 2809 2810 2811 2812

	/* WM1+ latency values stored in 0.5us units */
	if (level > 0) {
		pri_latency *= 5;
		spr_latency *= 5;
		cur_latency *= 5;
	}

2813
	if (pristate) {
2814
		result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2815
						     pri_latency, level);
2816
		result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2817 2818 2819
	}

	if (sprstate)
2820
		result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2821 2822

	if (curstate)
2823
		result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2824

2825 2826 2827
	result->enable = true;
}

2828
static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2829
				  u16 wm[8])
2830
{
2831 2832
	struct intel_uncore *uncore = &dev_priv->uncore;

2833
	if (INTEL_GEN(dev_priv) >= 9) {
2834
		u32 val;
2835
		int ret, i;
2836
		int level, max_level = ilk_wm_max_level(dev_priv);
2837 2838 2839 2840 2841

		/* read the first set of memory latencies[0:3] */
		val = 0; /* data0 to be programmed to 0 for first set */
		ret = sandybridge_pcode_read(dev_priv,
					     GEN9_PCODE_READ_MEM_LATENCY,
2842
					     &val, NULL);
2843 2844

		if (ret) {
2845 2846
			drm_err(&dev_priv->drm,
				"SKL Mailbox read error = %d\n", ret);
2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861
			return;
		}

		wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
		wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
				GEN9_MEM_LATENCY_LEVEL_MASK;
		wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
				GEN9_MEM_LATENCY_LEVEL_MASK;
		wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
				GEN9_MEM_LATENCY_LEVEL_MASK;

		/* read the second set of memory latencies[4:7] */
		val = 1; /* data0 to be programmed to 1 for second set */
		ret = sandybridge_pcode_read(dev_priv,
					     GEN9_PCODE_READ_MEM_LATENCY,
2862
					     &val, NULL);
2863
		if (ret) {
2864 2865
			drm_err(&dev_priv->drm,
				"SKL Mailbox read error = %d\n", ret);
2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876
			return;
		}

		wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
		wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
				GEN9_MEM_LATENCY_LEVEL_MASK;
		wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
				GEN9_MEM_LATENCY_LEVEL_MASK;
		wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
				GEN9_MEM_LATENCY_LEVEL_MASK;

2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889
		/*
		 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
		 * need to be disabled. We make sure to sanitize the values out
		 * of the punit to satisfy this requirement.
		 */
		for (level = 1; level <= max_level; level++) {
			if (wm[level] == 0) {
				for (i = level + 1; i <= max_level; i++)
					wm[i] = 0;
				break;
			}
		}

2890
		/*
2891
		 * WaWmMemoryReadLatency:skl+,glk
2892
		 *
2893
		 * punit doesn't take into account the read latency so we need
2894 2895
		 * to add 2us to the various latency levels we retrieve from the
		 * punit when level 0 response data us 0us.
2896
		 */
2897 2898 2899 2900 2901
		if (wm[0] == 0) {
			wm[0] += 2;
			for (level = 1; level <= max_level; level++) {
				if (wm[level] == 0)
					break;
2902
				wm[level] += 2;
2903
			}
2904 2905
		}

2906 2907 2908 2909 2910 2911
		/*
		 * WA Level-0 adjustment for 16GB DIMMs: SKL+
		 * If we could not get dimm info enable this WA to prevent from
		 * any underrun. If not able to get Dimm info assume 16GB dimm
		 * to avoid any underrun.
		 */
2912
		if (dev_priv->dram_info.is_16gb_dimm)
2913 2914
			wm[0] += 1;

2915
	} else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2916
		u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2917 2918 2919 2920

		wm[0] = (sskpd >> 56) & 0xFF;
		if (wm[0] == 0)
			wm[0] = sskpd & 0xF;
2921 2922 2923 2924
		wm[1] = (sskpd >> 4) & 0xFF;
		wm[2] = (sskpd >> 12) & 0xFF;
		wm[3] = (sskpd >> 20) & 0x1FF;
		wm[4] = (sskpd >> 32) & 0x1FF;
2925
	} else if (INTEL_GEN(dev_priv) >= 6) {
2926
		u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2927 2928 2929 2930 2931

		wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
		wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
		wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
		wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2932
	} else if (INTEL_GEN(dev_priv) >= 5) {
2933
		u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2934 2935 2936 2937 2938

		/* ILK primary LP0 latency is 700 ns */
		wm[0] = 7;
		wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
		wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2939 2940
	} else {
		MISSING_CASE(INTEL_DEVID(dev_priv));
2941 2942 2943
	}
}

2944
static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2945
				       u16 wm[5])
2946 2947
{
	/* ILK sprite LP0 latency is 1300 ns */
2948
	if (IS_GEN(dev_priv, 5))
2949 2950 2951
		wm[0] = 13;
}

2952
static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2953
				       u16 wm[5])
2954 2955
{
	/* ILK cursor LP0 latency is 1300 ns */
2956
	if (IS_GEN(dev_priv, 5))
2957 2958 2959
		wm[0] = 13;
}

2960
int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2961 2962
{
	/* how many WM levels are we expecting */
2963
	if (INTEL_GEN(dev_priv) >= 9)
2964
		return 7;
2965
	else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2966
		return 4;
2967
	else if (INTEL_GEN(dev_priv) >= 6)
2968
		return 3;
2969
	else
2970 2971
		return 2;
}
2972

2973
static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2974
				   const char *name,
2975
				   const u16 wm[8])
2976
{
2977
	int level, max_level = ilk_wm_max_level(dev_priv);
2978 2979 2980 2981 2982

	for (level = 0; level <= max_level; level++) {
		unsigned int latency = wm[level];

		if (latency == 0) {
2983 2984 2985
			drm_dbg_kms(&dev_priv->drm,
				    "%s WM%d latency not provided\n",
				    name, level);
2986 2987 2988
			continue;
		}

2989 2990 2991 2992
		/*
		 * - latencies are in us on gen9.
		 * - before then, WM1+ latency values are in 0.5us units
		 */
2993
		if (INTEL_GEN(dev_priv) >= 9)
2994 2995
			latency *= 10;
		else if (level > 0)
2996 2997
			latency *= 5;

2998 2999 3000
		drm_dbg_kms(&dev_priv->drm,
			    "%s WM%d latency %u (%u.%u usec)\n", name, level,
			    wm[level], latency / 10, latency % 10);
3001 3002 3003
	}
}

3004
static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3005
				    u16 wm[5], u16 min)
3006
{
3007
	int level, max_level = ilk_wm_max_level(dev_priv);
3008 3009 3010 3011 3012 3013

	if (wm[0] >= min)
		return false;

	wm[0] = max(wm[0], min);
	for (level = 1; level <= max_level; level++)
3014
		wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3015 3016 3017 3018

	return true;
}

3019
static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033
{
	bool changed;

	/*
	 * The BIOS provided WM memory latency values are often
	 * inadequate for high resolution displays. Adjust them.
	 */
	changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
		ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
		ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);

	if (!changed)
		return;

3034 3035
	drm_dbg_kms(&dev_priv->drm,
		    "WM latency values increased to avoid potential underruns\n");
3036 3037 3038
	intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3039 3040
}

3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062
static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
{
	/*
	 * On some SNB machines (Thinkpad X220 Tablet at least)
	 * LP3 usage can cause vblank interrupts to be lost.
	 * The DEIIR bit will go high but it looks like the CPU
	 * never gets interrupted.
	 *
	 * It's not clear whether other interrupt source could
	 * be affected or if this is somehow limited to vblank
	 * interrupts only. To play it safe we disable LP3
	 * watermarks entirely.
	 */
	if (dev_priv->wm.pri_latency[3] == 0 &&
	    dev_priv->wm.spr_latency[3] == 0 &&
	    dev_priv->wm.cur_latency[3] == 0)
		return;

	dev_priv->wm.pri_latency[3] = 0;
	dev_priv->wm.spr_latency[3] = 0;
	dev_priv->wm.cur_latency[3] = 0;

3063 3064
	drm_dbg_kms(&dev_priv->drm,
		    "LP3 watermarks disabled due to potential for lost interrupts\n");
3065 3066 3067 3068 3069
	intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
}

3070
static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3071
{
3072
	intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3073 3074 3075 3076 3077 3078

	memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
	       sizeof(dev_priv->wm.pri_latency));
	memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
	       sizeof(dev_priv->wm.pri_latency));

3079
	intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3080
	intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3081

3082 3083 3084
	intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3085

3086
	if (IS_GEN(dev_priv, 6)) {
3087
		snb_wm_latency_quirk(dev_priv);
3088 3089
		snb_wm_lp3_irq_quirk(dev_priv);
	}
3090 3091
}

3092
static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3093
{
3094
	intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3095
	intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3096 3097
}

3098
static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109
				 struct intel_pipe_wm *pipe_wm)
{
	/* LP0 watermark maximums depend on this pipe alone */
	const struct intel_wm_config config = {
		.num_pipes_active = 1,
		.sprites_enabled = pipe_wm->sprites_enabled,
		.sprites_scaled = pipe_wm->sprites_scaled,
	};
	struct ilk_wm_maximums max;

	/* LP0 watermarks always use 1/2 DDB partitioning */
3110
	ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3111 3112 3113

	/* At least LP0 must be valid */
	if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3114
		drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3115 3116 3117 3118 3119 3120
		return false;
	}

	return true;
}

3121
/* Compute new watermarks for the pipe */
3122
static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state)
3123
{
3124
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3125
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3126
	struct intel_pipe_wm *pipe_wm;
3127 3128
	struct intel_plane *plane;
	const struct intel_plane_state *plane_state;
3129 3130 3131
	const struct intel_plane_state *pristate = NULL;
	const struct intel_plane_state *sprstate = NULL;
	const struct intel_plane_state *curstate = NULL;
3132
	int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3133
	struct ilk_wm_maximums max;
3134

3135
	pipe_wm = &crtc_state->wm.ilk.optimal;
3136

3137 3138 3139 3140 3141 3142 3143
	intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
		if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
			pristate = plane_state;
		else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
			sprstate = plane_state;
		else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
			curstate = plane_state;
3144 3145
	}

3146
	pipe_wm->pipe_enabled = crtc_state->hw.active;
3147
	if (sprstate) {
3148 3149 3150 3151
		pipe_wm->sprites_enabled = sprstate->uapi.visible;
		pipe_wm->sprites_scaled = sprstate->uapi.visible &&
			(drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
			 drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
3152 3153
	}

3154 3155
	usable_level = max_level;

3156
	/* ILK/SNB: LP2+ watermarks only w/o sprites */
3157
	if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3158
		usable_level = 1;
3159 3160

	/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3161
	if (pipe_wm->sprites_scaled)
3162
		usable_level = 0;
3163

3164
	memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3165
	ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3166
			     pristate, sprstate, curstate, &pipe_wm->wm[0]);
3167

3168
	if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3169
		return -EINVAL;
3170

3171
	ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3172

3173 3174
	for (level = 1; level <= usable_level; level++) {
		struct intel_wm_level *wm = &pipe_wm->wm[level];
3175

3176
		ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3177
				     pristate, sprstate, curstate, wm);
3178 3179 3180 3181 3182 3183

		/*
		 * Disable any watermark level that exceeds the
		 * register maximums since such watermarks are
		 * always invalid.
		 */
3184 3185 3186 3187
		if (!ilk_validate_wm_level(level, &max, wm)) {
			memset(wm, 0, sizeof(*wm));
			break;
		}
3188 3189
	}

3190
	return 0;
3191 3192
}

3193 3194 3195 3196 3197
/*
 * Build a set of 'intermediate' watermark values that satisfy both the old
 * state and the new state.  These can be programmed to the hardware
 * immediately.
 */
3198
static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
3199
{
3200
	struct intel_crtc *intel_crtc = to_intel_crtc(newstate->uapi.crtc);
3201
	struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3202
	struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3203
	struct intel_atomic_state *intel_state =
3204
		to_intel_atomic_state(newstate->uapi.state);
3205 3206 3207
	const struct intel_crtc_state *oldstate =
		intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
	const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3208
	int level, max_level = ilk_wm_max_level(dev_priv);
3209 3210 3211 3212 3213 3214

	/*
	 * Start with the final, target watermarks, then combine with the
	 * currently active watermarks to get values that are safe both before
	 * and after the vblank.
	 */
3215
	*a = newstate->wm.ilk.optimal;
3216
	if (!newstate->hw.active || drm_atomic_crtc_needs_modeset(&newstate->uapi) ||
3217
	    intel_state->skip_intermediate_wm)
3218 3219
		return 0;

3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240
	a->pipe_enabled |= b->pipe_enabled;
	a->sprites_enabled |= b->sprites_enabled;
	a->sprites_scaled |= b->sprites_scaled;

	for (level = 0; level <= max_level; level++) {
		struct intel_wm_level *a_wm = &a->wm[level];
		const struct intel_wm_level *b_wm = &b->wm[level];

		a_wm->enable &= b_wm->enable;
		a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
		a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
		a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
		a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
	}

	/*
	 * We need to make sure that these merged watermark values are
	 * actually a valid configuration themselves.  If they're not,
	 * there's no safe way to transition from the old state to
	 * the new state, so we need to fail the atomic transaction.
	 */
3241
	if (!ilk_validate_pipe_wm(dev_priv, a))
3242 3243 3244 3245 3246 3247
		return -EINVAL;

	/*
	 * If our intermediate WM are identical to the final WM, then we can
	 * omit the post-vblank programming; only update if it's different.
	 */
3248 3249
	if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
		newstate->wm.need_postvbl_update = true;
3250 3251 3252 3253

	return 0;
}

3254 3255 3256
/*
 * Merge the watermarks from all active pipes for a specific level.
 */
3257
static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3258 3259 3260 3261 3262
			       int level,
			       struct intel_wm_level *ret_wm)
{
	const struct intel_crtc *intel_crtc;

3263 3264
	ret_wm->enable = true;

3265
	for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3266
		const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3267 3268 3269 3270
		const struct intel_wm_level *wm = &active->wm[level];

		if (!active->pipe_enabled)
			continue;
3271

3272 3273 3274 3275 3276
		/*
		 * The watermark values may have been used in the past,
		 * so we must maintain them in the registers for some
		 * time even if the level is now disabled.
		 */
3277
		if (!wm->enable)
3278
			ret_wm->enable = false;
3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289

		ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
		ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
		ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
		ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
	}
}

/*
 * Merge all low power watermarks for all active pipes.
 */
3290
static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3291
			 const struct intel_wm_config *config,
3292
			 const struct ilk_wm_maximums *max,
3293 3294
			 struct intel_pipe_wm *merged)
{
3295
	int level, max_level = ilk_wm_max_level(dev_priv);
3296
	int last_enabled_level = max_level;
3297

3298
	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3299
	if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3300
	    config->num_pipes_active > 1)
3301
		last_enabled_level = 0;
3302

3303
	/* ILK: FBC WM must be disabled always */
3304
	merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3305 3306 3307 3308 3309

	/* merge each WM1+ level */
	for (level = 1; level <= max_level; level++) {
		struct intel_wm_level *wm = &merged->wm[level];

3310
		ilk_merge_wm_level(dev_priv, level, wm);
3311

3312 3313 3314 3315 3316
		if (level > last_enabled_level)
			wm->enable = false;
		else if (!ilk_validate_wm_level(level, max, wm))
			/* make sure all following levels get disabled */
			last_enabled_level = level - 1;
3317 3318 3319 3320 3321 3322

		/*
		 * The spec says it is preferred to disable
		 * FBC WMs instead of disabling a WM level.
		 */
		if (wm->fbc_val > max->fbc) {
3323 3324
			if (wm->enable)
				merged->fbc_wm_enabled = false;
3325 3326 3327
			wm->fbc_val = 0;
		}
	}
3328 3329 3330 3331 3332 3333 3334

	/* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
	/*
	 * FIXME this is racy. FBC might get enabled later.
	 * What we should check here is whether FBC can be
	 * enabled sometime later.
	 */
3335
	if (IS_GEN(dev_priv, 5) && !merged->fbc_wm_enabled &&
3336
	    intel_fbc_is_active(dev_priv)) {
3337 3338 3339 3340 3341 3342
		for (level = 2; level <= max_level; level++) {
			struct intel_wm_level *wm = &merged->wm[level];

			wm->enable = false;
		}
	}
3343 3344
}

3345 3346 3347 3348 3349 3350
static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
{
	/* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
	return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
}

3351
/* The value we need to program into the WM_LPx latency field */
3352 3353
static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
				      int level)
3354
{
3355
	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3356 3357 3358 3359 3360
		return 2 * level;
	else
		return dev_priv->wm.pri_latency[level];
}

3361
static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3362
				   const struct intel_pipe_wm *merged,
3363
				   enum intel_ddb_partitioning partitioning,
3364
				   struct ilk_wm_values *results)
3365
{
3366 3367
	struct intel_crtc *intel_crtc;
	int level, wm_lp;
3368

3369
	results->enable_fbc_wm = merged->fbc_wm_enabled;
3370
	results->partitioning = partitioning;
3371

3372
	/* LP1+ register values */
3373
	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3374
		const struct intel_wm_level *r;
3375

3376
		level = ilk_wm_lp_to_level(wm_lp, merged);
3377

3378
		r = &merged->wm[level];
3379

3380 3381 3382 3383 3384
		/*
		 * Maintain the watermark values even if the level is
		 * disabled. Doing otherwise could cause underruns.
		 */
		results->wm_lp[wm_lp - 1] =
3385
			(ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3386 3387 3388
			(r->pri_val << WM1_LP_SR_SHIFT) |
			r->cur_val;

3389 3390 3391
		if (r->enable)
			results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;

3392
		if (INTEL_GEN(dev_priv) >= 8)
3393 3394 3395 3396 3397 3398
			results->wm_lp[wm_lp - 1] |=
				r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
		else
			results->wm_lp[wm_lp - 1] |=
				r->fbc_val << WM1_LP_FBC_SHIFT;

3399 3400 3401 3402
		/*
		 * Always set WM1S_LP_EN when spr_val != 0, even if the
		 * level is disabled. Doing otherwise could cause underruns.
		 */
3403
		if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3404
			drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3405 3406 3407
			results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
		} else
			results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3408
	}
3409

3410
	/* LP0 register values */
3411
	for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3412
		enum pipe pipe = intel_crtc->pipe;
3413 3414
		const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active.ilk;
		const struct intel_wm_level *r = &pipe_wm->wm[0];
3415

3416
		if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3417
			continue;
3418

3419 3420 3421 3422
		results->wm_pipe[pipe] =
			(r->pri_val << WM0_PIPE_PLANE_SHIFT) |
			(r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
			r->cur_val;
3423 3424 3425
	}
}

3426 3427
/* Find the result with the highest level enabled. Check for enable_fbc_wm in
 * case both are at the same level. Prefer r1 in case they're the same. */
3428 3429 3430 3431
static struct intel_pipe_wm *
ilk_find_best_result(struct drm_i915_private *dev_priv,
		     struct intel_pipe_wm *r1,
		     struct intel_pipe_wm *r2)
3432
{
3433
	int level, max_level = ilk_wm_max_level(dev_priv);
3434
	int level1 = 0, level2 = 0;
3435

3436 3437 3438 3439 3440
	for (level = 1; level <= max_level; level++) {
		if (r1->wm[level].enable)
			level1 = level;
		if (r2->wm[level].enable)
			level2 = level;
3441 3442
	}

3443 3444
	if (level1 == level2) {
		if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3445 3446 3447
			return r2;
		else
			return r1;
3448
	} else if (level1 > level2) {
3449 3450 3451 3452 3453 3454
		return r1;
	} else {
		return r2;
	}
}

3455 3456 3457 3458 3459 3460 3461
/* dirty bits used to track which watermarks need changes */
#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
#define WM_DIRTY_FBC (1 << 24)
#define WM_DIRTY_DDB (1 << 25)

3462
static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3463 3464
					 const struct ilk_wm_values *old,
					 const struct ilk_wm_values *new)
3465 3466 3467 3468 3469
{
	unsigned int dirty = 0;
	enum pipe pipe;
	int wm_lp;

3470
	for_each_pipe(dev_priv, pipe) {
3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507
		if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
			dirty |= WM_DIRTY_PIPE(pipe);
			/* Must disable LP1+ watermarks too */
			dirty |= WM_DIRTY_LP_ALL;
		}
	}

	if (old->enable_fbc_wm != new->enable_fbc_wm) {
		dirty |= WM_DIRTY_FBC;
		/* Must disable LP1+ watermarks too */
		dirty |= WM_DIRTY_LP_ALL;
	}

	if (old->partitioning != new->partitioning) {
		dirty |= WM_DIRTY_DDB;
		/* Must disable LP1+ watermarks too */
		dirty |= WM_DIRTY_LP_ALL;
	}

	/* LP1+ watermarks already deemed dirty, no need to continue */
	if (dirty & WM_DIRTY_LP_ALL)
		return dirty;

	/* Find the lowest numbered LP1+ watermark in need of an update... */
	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
		    old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
			break;
	}

	/* ...and mark it and all higher numbered LP1+ watermarks as dirty */
	for (; wm_lp <= 3; wm_lp++)
		dirty |= WM_DIRTY_LP(wm_lp);

	return dirty;
}

3508 3509
static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
			       unsigned int dirty)
3510
{
3511
	struct ilk_wm_values *previous = &dev_priv->wm.hw;
3512
	bool changed = false;
3513

3514 3515 3516
	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
		previous->wm_lp[2] &= ~WM1_LP_SR_EN;
		I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3517
		changed = true;
3518 3519 3520 3521
	}
	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
		previous->wm_lp[1] &= ~WM1_LP_SR_EN;
		I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3522
		changed = true;
3523 3524 3525 3526
	}
	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
		previous->wm_lp[0] &= ~WM1_LP_SR_EN;
		I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3527
		changed = true;
3528
	}
3529

3530 3531 3532 3533
	/*
	 * Don't touch WM1S_LP_EN here.
	 * Doing so could cause underruns.
	 */
3534

3535 3536 3537 3538 3539 3540 3541
	return changed;
}

/*
 * The spec says we shouldn't write when we don't need, because every write
 * causes WMs to be re-evaluated, expending some power.
 */
3542 3543
static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
				struct ilk_wm_values *results)
3544
{
3545
	struct ilk_wm_values *previous = &dev_priv->wm.hw;
3546
	unsigned int dirty;
3547
	u32 val;
3548

3549
	dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3550 3551 3552 3553 3554
	if (!dirty)
		return;

	_ilk_disable_lp_wm(dev_priv, dirty);

3555
	if (dirty & WM_DIRTY_PIPE(PIPE_A))
3556
		I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3557
	if (dirty & WM_DIRTY_PIPE(PIPE_B))
3558
		I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3559
	if (dirty & WM_DIRTY_PIPE(PIPE_C))
3560 3561
		I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);

3562
	if (dirty & WM_DIRTY_DDB) {
3563
		if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577
			val = I915_READ(WM_MISC);
			if (results->partitioning == INTEL_DDB_PART_1_2)
				val &= ~WM_MISC_DATA_PARTITION_5_6;
			else
				val |= WM_MISC_DATA_PARTITION_5_6;
			I915_WRITE(WM_MISC, val);
		} else {
			val = I915_READ(DISP_ARB_CTL2);
			if (results->partitioning == INTEL_DDB_PART_1_2)
				val &= ~DISP_DATA_PARTITION_5_6;
			else
				val |= DISP_DATA_PARTITION_5_6;
			I915_WRITE(DISP_ARB_CTL2, val);
		}
3578 3579
	}

3580
	if (dirty & WM_DIRTY_FBC) {
3581 3582 3583 3584 3585 3586 3587 3588
		val = I915_READ(DISP_ARB_CTL);
		if (results->enable_fbc_wm)
			val &= ~DISP_FBC_WM_DIS;
		else
			val |= DISP_FBC_WM_DIS;
		I915_WRITE(DISP_ARB_CTL, val);
	}

3589 3590 3591 3592
	if (dirty & WM_DIRTY_LP(1) &&
	    previous->wm_lp_spr[0] != results->wm_lp_spr[0])
		I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);

3593
	if (INTEL_GEN(dev_priv) >= 7) {
3594 3595 3596 3597 3598
		if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
			I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
		if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
			I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
	}
3599

3600
	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3601
		I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3602
	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3603
		I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3604
	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3605
		I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3606 3607

	dev_priv->wm.hw = *results;
3608 3609
}

3610
bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3611 3612 3613 3614
{
	return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
}

3615
u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3616
{
3617 3618 3619
	int i;
	int max_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
	u8 enabled_slices_mask = 0;
3620

3621 3622 3623 3624
	for (i = 0; i < max_slices; i++) {
		if (I915_READ(DBUF_CTL_S(i)) & DBUF_POWER_STATE)
			enabled_slices_mask |= BIT(i);
	}
3625

3626
	return enabled_slices_mask;
3627 3628
}

3629 3630 3631 3632
/*
 * FIXME: We still don't have the proper code detect if we need to apply the WA,
 * so assume we'll always need it in order to avoid underruns.
 */
3633
static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3634
{
3635
	return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv);
3636 3637
}

3638 3639 3640
static bool
intel_has_sagv(struct drm_i915_private *dev_priv)
{
3641 3642
	return (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) &&
		dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3643 3644
}

3645 3646 3647
static void
skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
{
3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659
	if (INTEL_GEN(dev_priv) >= 12) {
		u32 val = 0;
		int ret;

		ret = sandybridge_pcode_read(dev_priv,
					     GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
					     &val, NULL);
		if (!ret) {
			dev_priv->sagv_block_time_us = val;
			return;
		}

3660
		drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3661
	} else if (IS_GEN(dev_priv, 11)) {
3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677
		dev_priv->sagv_block_time_us = 10;
		return;
	} else if (IS_GEN(dev_priv, 10)) {
		dev_priv->sagv_block_time_us = 20;
		return;
	} else if (IS_GEN(dev_priv, 9)) {
		dev_priv->sagv_block_time_us = 30;
		return;
	} else {
		MISSING_CASE(INTEL_GEN(dev_priv));
	}

	/* Default to an unusable block time */
	dev_priv->sagv_block_time_us = -1;
}

3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689
/*
 * SAGV dynamically adjusts the system agent voltage and clock frequencies
 * depending on power and performance requirements. The display engine access
 * to system memory is blocked during the adjustment time. Because of the
 * blocking time, having this enabled can cause full system hangs and/or pipe
 * underruns if we don't meet all of the following requirements:
 *
 *  - <= 1 pipe enabled
 *  - All planes can enable watermarks for latencies >= SAGV engine block time
 *  - We're not using an interlaced display configuration
 */
int
3690
intel_enable_sagv(struct drm_i915_private *dev_priv)
3691 3692 3693
{
	int ret;

3694 3695 3696 3697
	if (!intel_has_sagv(dev_priv))
		return 0;

	if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3698 3699
		return 0;

3700
	drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3701 3702 3703
	ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
				      GEN9_SAGV_ENABLE);

3704
	/* We don't need to wait for SAGV when enabling */
3705 3706 3707

	/*
	 * Some skl systems, pre-release machines in particular,
3708
	 * don't actually have SAGV.
3709
	 */
3710
	if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3711
		drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3712
		dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3713 3714
		return 0;
	} else if (ret < 0) {
3715
		drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3716 3717 3718
		return ret;
	}

3719
	dev_priv->sagv_status = I915_SAGV_ENABLED;
3720 3721 3722 3723
	return 0;
}

int
3724
intel_disable_sagv(struct drm_i915_private *dev_priv)
3725
{
3726
	int ret;
3727

3728 3729 3730 3731
	if (!intel_has_sagv(dev_priv))
		return 0;

	if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3732 3733
		return 0;

3734
	drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3735
	/* bspec says to keep retrying for at least 1 ms */
3736 3737 3738 3739
	ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
				GEN9_SAGV_DISABLE,
				GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
				1);
3740 3741
	/*
	 * Some skl systems, pre-release machines in particular,
3742
	 * don't actually have SAGV.
3743
	 */
3744
	if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3745
		drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3746
		dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3747
		return 0;
3748
	} else if (ret < 0) {
3749
		drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3750
		return ret;
3751 3752
	}

3753
	dev_priv->sagv_status = I915_SAGV_DISABLED;
3754 3755 3756
	return 0;
}

3757 3758 3759
void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
{
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3760
	const struct intel_bw_state *new_bw_state;
3761 3762
	const struct intel_bw_state *old_bw_state;
	u32 new_mask = 0;
3763

3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777
	/*
	 * Just return if we can't control SAGV or don't have it.
	 * This is different from situation when we have SAGV but just can't
	 * afford it due to DBuf limitation - in case if SAGV is completely
	 * disabled in a BIOS, we are not even allowed to send a PCode request,
	 * as it will throw an error. So have to check it here.
	 */
	if (!intel_has_sagv(dev_priv))
		return;

	new_bw_state = intel_atomic_get_new_bw_state(state);
	if (!new_bw_state)
		return;

3778
	if (INTEL_GEN(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) {
3779
		intel_disable_sagv(dev_priv);
3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805
		return;
	}

	old_bw_state = intel_atomic_get_old_bw_state(state);
	/*
	 * Nothing to mask
	 */
	if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
		return;

	new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;

	/*
	 * If new mask is zero - means there is nothing to mask,
	 * we can only unmask, which should be done in unmask.
	 */
	if (!new_mask)
		return;

	/*
	 * Restrict required qgv points before updating the configuration.
	 * According to BSpec we can't mask and unmask qgv points at the same
	 * time. Also masking should be done before updating the configuration
	 * and unmasking afterwards.
	 */
	icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3806 3807 3808 3809 3810
}

void intel_sagv_post_plane_update(struct intel_atomic_state *state)
{
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3811
	const struct intel_bw_state *new_bw_state;
3812 3813
	const struct intel_bw_state *old_bw_state;
	u32 new_mask = 0;
3814 3815 3816 3817 3818 3819 3820 3821 3822 3823

	/*
	 * Just return if we can't control SAGV or don't have it.
	 * This is different from situation when we have SAGV but just can't
	 * afford it due to DBuf limitation - in case if SAGV is completely
	 * disabled in a BIOS, we are not even allowed to send a PCode request,
	 * as it will throw an error. So have to check it here.
	 */
	if (!intel_has_sagv(dev_priv))
		return;
3824

3825 3826 3827 3828
	new_bw_state = intel_atomic_get_new_bw_state(state);
	if (!new_bw_state)
		return;

3829
	if (INTEL_GEN(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) {
3830
		intel_enable_sagv(dev_priv);
3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849
		return;
	}

	old_bw_state = intel_atomic_get_old_bw_state(state);
	/*
	 * Nothing to unmask
	 */
	if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
		return;

	new_mask = new_bw_state->qgv_points_mask;

	/*
	 * Allow required qgv points after updating the configuration.
	 * According to BSpec we can't mask and unmask qgv points at the same
	 * time. Also masking should be done before updating the configuration
	 * and unmasking afterwards.
	 */
	icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3850 3851
}

3852
static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3853
{
3854
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3855
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3856
	struct intel_plane *plane;
3857
	const struct intel_plane_state *plane_state;
3858
	int level, latency;
3859

3860 3861 3862
	if (!intel_has_sagv(dev_priv))
		return false;

3863
	if (!crtc_state->hw.active)
3864
		return true;
3865

3866
	if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3867 3868
		return false;

3869
	intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3870
		const struct skl_plane_wm *wm =
3871
			&crtc_state->wm.skl.optimal.planes[plane->id];
3872

3873
		/* Skip this plane if it's not enabled */
3874
		if (!wm->wm[0].plane_en)
3875 3876 3877
			continue;

		/* Find the highest enabled wm level for this plane */
3878
		for (level = ilk_wm_max_level(dev_priv);
3879
		     !wm->wm[level].plane_en; --level)
3880 3881
		     { }

3882 3883
		latency = dev_priv->wm.skl_latency[level];

3884
		if (skl_needs_memory_bw_wa(dev_priv) &&
3885
		    plane_state->uapi.fb->modifier ==
3886 3887 3888
		    I915_FORMAT_MOD_X_TILED)
			latency += 15;

3889
		/*
3890 3891
		 * If any of the planes on this pipe don't enable wm levels that
		 * incur memory latencies higher than sagv_block_time_us we
3892
		 * can't enable SAGV.
3893
		 */
3894
		if (latency < dev_priv->sagv_block_time_us)
3895 3896 3897 3898 3899 3900
			return false;
	}

	return true;
}

3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921
static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
{
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
	enum plane_id plane_id;

	if (!crtc_state->hw.active)
		return true;

	for_each_plane_id_on_crtc(crtc, plane_id) {
		const struct skl_ddb_entry *plane_alloc =
			&crtc_state->wm.skl.plane_ddb_y[plane_id];
		const struct skl_plane_wm *wm =
			&crtc_state->wm.skl.optimal.planes[plane_id];

		if (skl_ddb_entry_size(plane_alloc) < wm->sagv_wm0.min_ddb_alloc)
			return false;
	}

	return true;
}

3922 3923
static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
{
3924 3925 3926 3927 3928 3929 3930
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);

	if (INTEL_GEN(dev_priv) >= 12)
		return tgl_crtc_can_enable_sagv(crtc_state);
	else
		return skl_crtc_can_enable_sagv(crtc_state);
3931 3932
}

3933 3934
bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
			   const struct intel_bw_state *bw_state)
3935
{
3936 3937
	if (INTEL_GEN(dev_priv) < 11 &&
	    bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
3938 3939
		return false;

3940 3941 3942 3943 3944
	return bw_state->pipe_sagv_reject == 0;
}

static int intel_compute_sagv_mask(struct intel_atomic_state *state)
{
3945
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3946
	int ret;
3947
	struct intel_crtc *crtc;
3948
	struct intel_crtc_state *new_crtc_state;
3949 3950 3951
	struct intel_bw_state *new_bw_state = NULL;
	const struct intel_bw_state *old_bw_state = NULL;
	int i;
3952

3953 3954 3955 3956 3957
	for_each_new_intel_crtc_in_state(state, crtc,
					 new_crtc_state, i) {
		new_bw_state = intel_atomic_get_bw_state(state);
		if (IS_ERR(new_bw_state))
			return PTR_ERR(new_bw_state);
3958

3959
		old_bw_state = intel_atomic_get_old_bw_state(state);
3960

3961 3962 3963 3964 3965
		if (intel_crtc_can_enable_sagv(new_crtc_state))
			new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
		else
			new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
	}
3966

3967 3968
	if (!new_bw_state)
		return 0;
3969

3970 3971
	new_bw_state->active_pipes =
		intel_calc_active_pipes(state, old_bw_state->active_pipes);
3972

3973 3974 3975 3976 3977 3978
	if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
		ret = intel_atomic_lock_global_state(&new_bw_state->base);
		if (ret)
			return ret;
	}

3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992
	for_each_new_intel_crtc_in_state(state, crtc,
					 new_crtc_state, i) {
		struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;

		/*
		 * We store use_sagv_wm in the crtc state rather than relying on
		 * that bw state since we have no convenient way to get at the
		 * latter from the plane commit hooks (especially in the legacy
		 * cursor case)
		 */
		pipe_wm->use_sagv_wm = INTEL_GEN(dev_priv) >= 12 &&
				       intel_can_enable_sagv(dev_priv, new_bw_state);
	}

3993 3994
	if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
	    intel_can_enable_sagv(dev_priv, old_bw_state)) {
3995 3996 3997 3998 3999 4000 4001 4002 4003 4004
		ret = intel_atomic_serialize_global_state(&new_bw_state->base);
		if (ret)
			return ret;
	} else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
		ret = intel_atomic_lock_global_state(&new_bw_state->base);
		if (ret)
			return ret;
	}

	return 0;
4005 4006
}

4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028
/*
 * Calculate initial DBuf slice offset, based on slice size
 * and mask(i.e if slice size is 1024 and second slice is enabled
 * offset would be 1024)
 */
static unsigned int
icl_get_first_dbuf_slice_offset(u32 dbuf_slice_mask,
				u32 slice_size,
				u32 ddb_size)
{
	unsigned int offset = 0;

	if (!dbuf_slice_mask)
		return 0;

	offset = (ffs(dbuf_slice_mask) - 1) * slice_size;

	WARN_ON(offset >= ddb_size);
	return offset;
}

static u16 intel_get_ddb_size(struct drm_i915_private *dev_priv)
4029 4030 4031
{
	u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;

4032
	drm_WARN_ON(&dev_priv->drm, ddb_size == 0);
4033 4034 4035 4036 4037 4038 4039

	if (INTEL_GEN(dev_priv) < 11)
		return ddb_size - 4; /* 4 blocks for bypass path allocation */

	return ddb_size;
}

4040
static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4041
				  u8 active_pipes);
4042

4043
static void
4044
skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv,
4045
				   const struct intel_crtc_state *crtc_state,
4046
				   const u64 total_data_rate,
4047 4048
				   struct skl_ddb_entry *alloc, /* out */
				   int *num_active /* out */)
4049
{
4050
	struct drm_atomic_state *state = crtc_state->uapi.state;
4051
	struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
4052
	struct drm_crtc *for_crtc = crtc_state->uapi.crtc;
4053
	const struct intel_crtc *crtc;
4054
	u32 pipe_width = 0, total_width_in_range = 0, width_before_pipe_in_range = 0;
4055 4056
	enum pipe for_pipe = to_intel_crtc(for_crtc)->pipe;
	u16 ddb_size;
4057
	u32 ddb_range_size;
4058
	u32 i;
4059 4060 4061 4062 4063 4064
	u32 dbuf_slice_mask;
	u32 active_pipes;
	u32 offset;
	u32 slice_size;
	u32 total_slice_mask;
	u32 start, end;
4065

4066
	if (drm_WARN_ON(&dev_priv->drm, !state) || !crtc_state->hw.active) {
4067 4068
		alloc->start = 0;
		alloc->end = 0;
4069
		*num_active = hweight8(dev_priv->active_pipes);
4070 4071 4072
		return;
	}

4073
	if (intel_state->active_pipe_changes)
4074
		active_pipes = intel_state->active_pipes;
4075
	else
4076 4077 4078 4079 4080
		active_pipes = dev_priv->active_pipes;

	*num_active = hweight8(active_pipes);

	ddb_size = intel_get_ddb_size(dev_priv);
4081

4082
	slice_size = ddb_size / INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4083

4084
	/*
4085 4086 4087 4088 4089 4090
	 * If the state doesn't change the active CRTC's or there is no
	 * modeset request, then there's no need to recalculate;
	 * the existing pipe allocation limits should remain unchanged.
	 * Note that we're safe from racing commits since any racing commit
	 * that changes the active CRTC list or do modeset would need to
	 * grab _all_ crtc locks, including the one we currently hold.
4091
	 */
4092
	if (!intel_state->active_pipe_changes && !intel_state->modeset) {
4093 4094 4095 4096 4097
		/*
		 * alloc may be cleared by clear_intel_crtc_state,
		 * copy from old state to be sure
		 */
		*alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
4098
		return;
4099
	}
4100

4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124
	/*
	 * Get allowed DBuf slices for correspondent pipe and platform.
	 */
	dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state, active_pipes);

	DRM_DEBUG_KMS("DBuf slice mask %x pipe %c active pipes %x\n",
		      dbuf_slice_mask,
		      pipe_name(for_pipe), active_pipes);

	/*
	 * Figure out at which DBuf slice we start, i.e if we start at Dbuf S2
	 * and slice size is 1024, the offset would be 1024
	 */
	offset = icl_get_first_dbuf_slice_offset(dbuf_slice_mask,
						 slice_size, ddb_size);

	/*
	 * Figure out total size of allowed DBuf slices, which is basically
	 * a number of allowed slices for that pipe multiplied by slice size.
	 * Inside of this
	 * range ddb entries are still allocated in proportion to display width.
	 */
	ddb_range_size = hweight8(dbuf_slice_mask) * slice_size;

4125 4126 4127 4128 4129
	/*
	 * Watermark/ddb requirement highly depends upon width of the
	 * framebuffer, So instead of allocating DDB equally among pipes
	 * distribute DDB based on resolution/width of the display.
	 */
4130
	total_slice_mask = dbuf_slice_mask;
4131 4132
	for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
		const struct drm_display_mode *adjusted_mode =
4133
			&crtc_state->hw.adjusted_mode;
4134
		enum pipe pipe = crtc->pipe;
4135
		int hdisplay, vdisplay;
4136
		u32 pipe_dbuf_slice_mask;
4137

4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160
		if (!crtc_state->hw.active)
			continue;

		pipe_dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state,
							       active_pipes);

		/*
		 * According to BSpec pipe can share one dbuf slice with another
		 * pipes or pipe can use multiple dbufs, in both cases we
		 * account for other pipes only if they have exactly same mask.
		 * However we need to account how many slices we should enable
		 * in total.
		 */
		total_slice_mask |= pipe_dbuf_slice_mask;

		/*
		 * Do not account pipes using other slice sets
		 * luckily as of current BSpec slice sets do not partially
		 * intersect(pipes share either same one slice or same slice set
		 * i.e no partial intersection), so it is enough to check for
		 * equality for now.
		 */
		if (dbuf_slice_mask != pipe_dbuf_slice_mask)
4161 4162 4163
			continue;

		drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay);
4164 4165

		total_width_in_range += hdisplay;
4166 4167

		if (pipe < for_pipe)
4168
			width_before_pipe_in_range += hdisplay;
4169 4170 4171 4172
		else if (pipe == for_pipe)
			pipe_width = hdisplay;
	}

4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190
	/*
	 * FIXME: For now we always enable slice S1 as per
	 * the Bspec display initialization sequence.
	 */
	intel_state->enabled_dbuf_slices_mask = total_slice_mask | BIT(DBUF_S1);

	start = ddb_range_size * width_before_pipe_in_range / total_width_in_range;
	end = ddb_range_size *
		(width_before_pipe_in_range + pipe_width) / total_width_in_range;

	alloc->start = offset + start;
	alloc->end = offset + end;

	DRM_DEBUG_KMS("Pipe %d ddb %d-%d\n", for_pipe,
		      alloc->start, alloc->end);
	DRM_DEBUG_KMS("Enabled ddb slices mask %x num supported %d\n",
		      intel_state->enabled_dbuf_slices_mask,
		      INTEL_INFO(dev_priv)->num_supported_dbuf_slices);
4191 4192
}

4193 4194 4195 4196 4197
static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
				 int width, const struct drm_format_info *format,
				 u64 modifier, unsigned int rotation,
				 u32 plane_pixel_rate, struct skl_wm_params *wp,
				 int color_plane);
4198
static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4199
				 int level,
4200
				 unsigned int latency,
4201 4202 4203 4204 4205 4206 4207
				 const struct skl_wm_params *wp,
				 const struct skl_wm_level *result_prev,
				 struct skl_wm_level *result /* out */);

static unsigned int
skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
		      int num_active)
4208
{
4209
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4210 4211 4212 4213 4214 4215 4216 4217 4218 4219
	int level, max_level = ilk_wm_max_level(dev_priv);
	struct skl_wm_level wm = {};
	int ret, min_ddb_alloc = 0;
	struct skl_wm_params wp;

	ret = skl_compute_wm_params(crtc_state, 256,
				    drm_format_info(DRM_FORMAT_ARGB8888),
				    DRM_FORMAT_MOD_LINEAR,
				    DRM_MODE_ROTATE_0,
				    crtc_state->pixel_rate, &wp, 0);
4220
	drm_WARN_ON(&dev_priv->drm, ret);
4221 4222

	for (level = 0; level <= max_level; level++) {
4223 4224 4225
		unsigned int latency = dev_priv->wm.skl_latency[level];

		skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm);
4226 4227 4228 4229 4230
		if (wm.min_ddb_alloc == U16_MAX)
			break;

		min_ddb_alloc = wm.min_ddb_alloc;
	}
4231

4232
	return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4233 4234
}

4235 4236
static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
				       struct skl_ddb_entry *entry, u32 reg)
4237
{
4238

4239 4240
	entry->start = reg & DDB_ENTRY_MASK;
	entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
4241

4242 4243
	if (entry->end)
		entry->end += 1;
4244 4245
}

4246 4247 4248 4249
static void
skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
			   const enum pipe pipe,
			   const enum plane_id plane_id,
4250 4251
			   struct skl_ddb_entry *ddb_y,
			   struct skl_ddb_entry *ddb_uv)
4252
{
4253 4254
	u32 val, val2;
	u32 fourcc = 0;
4255 4256 4257 4258

	/* Cursor doesn't support NV12/planar, so no extra calculation needed */
	if (plane_id == PLANE_CURSOR) {
		val = I915_READ(CUR_BUF_CFG(pipe));
4259
		skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4260 4261 4262 4263 4264 4265
		return;
	}

	val = I915_READ(PLANE_CTL(pipe, plane_id));

	/* No DDB allocated for disabled planes */
4266 4267 4268 4269
	if (val & PLANE_CTL_ENABLE)
		fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
					      val & PLANE_CTL_ORDER_RGBX,
					      val & PLANE_CTL_ALPHA_MASK);
4270

4271 4272 4273 4274 4275
	if (INTEL_GEN(dev_priv) >= 11) {
		val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
		skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
	} else {
		val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4276
		val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
4277

4278 4279
		if (fourcc &&
		    drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4280 4281 4282 4283
			swap(val, val2);

		skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
		skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4284 4285 4286
	}
}

4287 4288 4289
void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
			       struct skl_ddb_entry *ddb_y,
			       struct skl_ddb_entry *ddb_uv)
4290
{
4291 4292 4293
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	enum intel_display_power_domain power_domain;
	enum pipe pipe = crtc->pipe;
4294
	intel_wakeref_t wakeref;
4295
	enum plane_id plane_id;
4296

4297
	power_domain = POWER_DOMAIN_PIPE(pipe);
4298 4299
	wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
	if (!wakeref)
4300
		return;
4301

4302 4303 4304 4305 4306
	for_each_plane_id_on_crtc(crtc, plane_id)
		skl_ddb_get_hw_plane_state(dev_priv, pipe,
					   plane_id,
					   &ddb_y[plane_id],
					   &ddb_uv[plane_id]);
4307

4308
	intel_display_power_put(dev_priv, power_domain, wakeref);
4309
}
4310

4311
void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv)
4312
{
4313 4314
	dev_priv->enabled_dbuf_slices_mask =
				intel_enabled_dbuf_slices_mask(dev_priv);
4315 4316
}

4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332
/*
 * Determines the downscale amount of a plane for the purposes of watermark calculations.
 * The bspec defines downscale amount as:
 *
 * """
 * Horizontal down scale amount = maximum[1, Horizontal source size /
 *                                           Horizontal destination size]
 * Vertical down scale amount = maximum[1, Vertical source size /
 *                                         Vertical destination size]
 * Total down scale amount = Horizontal down scale amount *
 *                           Vertical down scale amount
 * """
 *
 * Return value is provided in 16.16 fixed point form to retain fractional part.
 * Caller should take care of dividing & rounding off the value.
 */
4333
static uint_fixed_16_16_t
4334 4335
skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
			   const struct intel_plane_state *plane_state)
4336
{
4337
	u32 src_w, src_h, dst_w, dst_h;
4338 4339
	uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
	uint_fixed_16_16_t downscale_h, downscale_w;
4340

4341
	if (WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state)))
4342
		return u32_to_fixed16(0);
4343

4344 4345 4346 4347 4348 4349 4350
	/*
	 * Src coordinates are already rotated by 270 degrees for
	 * the 90/270 degree plane rotation cases (to match the
	 * GTT mapping), hence no need to account for rotation here.
	 *
	 * n.b., src is 16.16 fixed point, dst is whole integer.
	 */
4351 4352 4353 4354
	src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
	src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
	dst_w = drm_rect_width(&plane_state->uapi.dst);
	dst_h = drm_rect_height(&plane_state->uapi.dst);
4355

4356 4357 4358 4359
	fp_w_ratio = div_fixed16(src_w, dst_w);
	fp_h_ratio = div_fixed16(src_h, dst_h);
	downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
	downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4360

4361
	return mul_fixed16(downscale_w, downscale_h);
4362 4363
}

4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378
struct dbuf_slice_conf_entry {
	u8 active_pipes;
	u8 dbuf_mask[I915_MAX_PIPES];
};

/*
 * Table taken from Bspec 12716
 * Pipes do have some preferred DBuf slice affinity,
 * plus there are some hardcoded requirements on how
 * those should be distributed for multipipe scenarios.
 * For more DBuf slices algorithm can get even more messy
 * and less readable, so decided to use a table almost
 * as is from BSpec itself - that way it is at least easier
 * to compare, change and check.
 */
4379
static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4380 4381 4382 4383 4384
/* Autogenerated with igt/tools/intel_dbuf_map tool: */
{
	{
		.active_pipes = BIT(PIPE_A),
		.dbuf_mask = {
4385 4386
			[PIPE_A] = BIT(DBUF_S1),
		},
4387 4388 4389 4390
	},
	{
		.active_pipes = BIT(PIPE_B),
		.dbuf_mask = {
4391 4392
			[PIPE_B] = BIT(DBUF_S1),
		},
4393 4394 4395 4396 4397
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
4398 4399
			[PIPE_B] = BIT(DBUF_S2),
		},
4400 4401 4402 4403
	},
	{
		.active_pipes = BIT(PIPE_C),
		.dbuf_mask = {
4404 4405
			[PIPE_C] = BIT(DBUF_S2),
		},
4406 4407 4408 4409 4410
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
4411 4412
			[PIPE_C] = BIT(DBUF_S2),
		},
4413 4414 4415 4416 4417
	},
	{
		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_B] = BIT(DBUF_S1),
4418 4419
			[PIPE_C] = BIT(DBUF_S2),
		},
4420 4421 4422 4423 4424 4425
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
			[PIPE_B] = BIT(DBUF_S1),
4426 4427
			[PIPE_C] = BIT(DBUF_S2),
		},
4428
	},
4429
	{}
4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441
};

/*
 * Table taken from Bspec 49255
 * Pipes do have some preferred DBuf slice affinity,
 * plus there are some hardcoded requirements on how
 * those should be distributed for multipipe scenarios.
 * For more DBuf slices algorithm can get even more messy
 * and less readable, so decided to use a table almost
 * as is from BSpec itself - that way it is at least easier
 * to compare, change and check.
 */
4442
static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4443 4444 4445 4446 4447
/* Autogenerated with igt/tools/intel_dbuf_map tool: */
{
	{
		.active_pipes = BIT(PIPE_A),
		.dbuf_mask = {
4448 4449
			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
		},
4450 4451 4452 4453
	},
	{
		.active_pipes = BIT(PIPE_B),
		.dbuf_mask = {
4454 4455
			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
		},
4456 4457 4458 4459 4460
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S2),
4461 4462
			[PIPE_B] = BIT(DBUF_S1),
		},
4463 4464 4465 4466
	},
	{
		.active_pipes = BIT(PIPE_C),
		.dbuf_mask = {
4467 4468
			[PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
		},
4469 4470 4471 4472 4473
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
4474 4475
			[PIPE_C] = BIT(DBUF_S2),
		},
4476 4477 4478 4479 4480
	},
	{
		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_B] = BIT(DBUF_S1),
4481 4482
			[PIPE_C] = BIT(DBUF_S2),
		},
4483 4484 4485 4486 4487 4488
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
			[PIPE_B] = BIT(DBUF_S1),
4489 4490
			[PIPE_C] = BIT(DBUF_S2),
		},
4491 4492 4493 4494
	},
	{
		.active_pipes = BIT(PIPE_D),
		.dbuf_mask = {
4495 4496
			[PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
		},
4497 4498 4499 4500 4501
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
4502 4503
			[PIPE_D] = BIT(DBUF_S2),
		},
4504 4505 4506 4507 4508
	},
	{
		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_B] = BIT(DBUF_S1),
4509 4510
			[PIPE_D] = BIT(DBUF_S2),
		},
4511 4512 4513 4514 4515 4516
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
			[PIPE_B] = BIT(DBUF_S1),
4517 4518
			[PIPE_D] = BIT(DBUF_S2),
		},
4519 4520 4521 4522 4523
	},
	{
		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_C] = BIT(DBUF_S1),
4524 4525
			[PIPE_D] = BIT(DBUF_S2),
		},
4526 4527 4528 4529 4530 4531
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
			[PIPE_C] = BIT(DBUF_S2),
4532 4533
			[PIPE_D] = BIT(DBUF_S2),
		},
4534 4535 4536 4537 4538 4539
	},
	{
		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_B] = BIT(DBUF_S1),
			[PIPE_C] = BIT(DBUF_S2),
4540 4541
			[PIPE_D] = BIT(DBUF_S2),
		},
4542 4543 4544 4545 4546 4547 4548
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
			[PIPE_B] = BIT(DBUF_S1),
			[PIPE_C] = BIT(DBUF_S2),
4549 4550
			[PIPE_D] = BIT(DBUF_S2),
		},
4551
	},
4552
	{}
4553 4554
};

4555 4556
static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
			      const struct dbuf_slice_conf_entry *dbuf_slices)
4557 4558 4559
{
	int i;

4560
	for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571
		if (dbuf_slices[i].active_pipes == active_pipes)
			return dbuf_slices[i].dbuf_mask[pipe];
	}
	return 0;
}

/*
 * This function finds an entry with same enabled pipe configuration and
 * returns correspondent DBuf slice mask as stated in BSpec for particular
 * platform.
 */
4572
static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585
{
	/*
	 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
	 * required calculating "pipe ratio" in order to determine
	 * if one or two slices can be used for single pipe configurations
	 * as additional constraint to the existing table.
	 * However based on recent info, it should be not "pipe ratio"
	 * but rather ratio between pixel_rate and cdclk with additional
	 * constants, so for now we are using only table until this is
	 * clarified. Also this is the reason why crtc_state param is
	 * still here - we will need it once those additional constraints
	 * pop up.
	 */
4586
	return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
4587 4588
}

4589
static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4590
{
4591
	return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
4592 4593 4594
}

static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4595
				  u8 active_pipes)
4596 4597 4598 4599 4600 4601
{
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	enum pipe pipe = crtc->pipe;

	if (IS_GEN(dev_priv, 12))
4602
		return tgl_compute_dbuf_slices(pipe, active_pipes);
4603
	else if (IS_GEN(dev_priv, 11))
4604
		return icl_compute_dbuf_slices(pipe, active_pipes);
4605 4606 4607 4608 4609 4610 4611
	/*
	 * For anything else just return one slice yet.
	 * Should be extended for other platforms.
	 */
	return BIT(DBUF_S1);
}

4612
static u64
4613 4614
skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
			     const struct intel_plane_state *plane_state,
4615
			     int color_plane)
4616
{
4617
	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4618
	const struct drm_framebuffer *fb = plane_state->hw.fb;
4619 4620
	u32 data_rate;
	u32 width = 0, height = 0;
4621
	uint_fixed_16_16_t down_scale_amount;
4622
	u64 rate;
4623

4624
	if (!plane_state->uapi.visible)
4625
		return 0;
4626

4627
	if (plane->id == PLANE_CURSOR)
4628
		return 0;
4629 4630

	if (color_plane == 1 &&
4631
	    !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4632
		return 0;
4633

4634 4635 4636 4637 4638
	/*
	 * Src coordinates are already rotated by 270 degrees for
	 * the 90/270 degree plane rotation cases (to match the
	 * GTT mapping), hence no need to account for rotation here.
	 */
4639 4640
	width = drm_rect_width(&plane_state->uapi.src) >> 16;
	height = drm_rect_height(&plane_state->uapi.src) >> 16;
4641

4642
	/* UV plane does 1/2 pixel sub-sampling */
4643
	if (color_plane == 1) {
4644 4645
		width /= 2;
		height /= 2;
4646 4647
	}

4648
	data_rate = width * height;
4649

4650
	down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4651

4652 4653
	rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);

4654
	rate *= fb->format->cpp[color_plane];
4655
	return rate;
4656 4657
}

4658
static u64
4659
skl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4660 4661
				 u64 *plane_data_rate,
				 u64 *uv_plane_data_rate)
4662
{
4663 4664
	struct intel_plane *plane;
	const struct intel_plane_state *plane_state;
4665
	u64 total_data_rate = 0;
4666

4667
	/* Calculate and cache data rate for each plane */
4668 4669
	intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
		enum plane_id plane_id = plane->id;
4670
		u64 rate;
4671

4672
		/* packed/y */
4673
		rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4674
		plane_data_rate[plane_id] = rate;
4675
		total_data_rate += rate;
4676

4677
		/* uv-plane */
4678
		rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4679
		uv_plane_data_rate[plane_id] = rate;
4680
		total_data_rate += rate;
4681 4682 4683 4684 4685
	}

	return total_data_rate;
}

4686
static u64
4687
icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4688 4689
				 u64 *plane_data_rate)
{
4690 4691
	struct intel_plane *plane;
	const struct intel_plane_state *plane_state;
4692 4693 4694
	u64 total_data_rate = 0;

	/* Calculate and cache data rate for each plane */
4695 4696
	intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
		enum plane_id plane_id = plane->id;
4697 4698
		u64 rate;

4699
		if (!plane_state->planar_linked_plane) {
4700
			rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4701 4702 4703 4704 4705 4706 4707
			plane_data_rate[plane_id] = rate;
			total_data_rate += rate;
		} else {
			enum plane_id y_plane_id;

			/*
			 * The slave plane might not iterate in
4708
			 * intel_atomic_crtc_state_for_each_plane_state(),
4709 4710 4711 4712
			 * and needs the master plane state which may be
			 * NULL if we try get_new_plane_state(), so we
			 * always calculate from the master.
			 */
4713
			if (plane_state->planar_slave)
4714 4715 4716
				continue;

			/* Y plane rate is calculated on the slave */
4717
			rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4718
			y_plane_id = plane_state->planar_linked_plane->id;
4719 4720 4721
			plane_data_rate[y_plane_id] = rate;
			total_data_rate += rate;

4722
			rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4723 4724 4725 4726 4727 4728 4729 4730
			plane_data_rate[plane_id] = rate;
			total_data_rate += rate;
		}
	}

	return total_data_rate;
}

4731 4732 4733 4734 4735
static const struct skl_wm_level *
skl_plane_wm_level(const struct intel_crtc_state *crtc_state,
		   enum plane_id plane_id,
		   int level)
{
4736 4737 4738 4739 4740
	const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];

	if (level == 0 && pipe_wm->use_sagv_wm)
		return &wm->sagv_wm0;
4741 4742 4743 4744

	return &wm->wm[level];
}

4745
static int
4746
skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state)
4747
{
4748 4749
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4750
	struct skl_ddb_entry *alloc = &crtc_state->wm.skl.ddb;
4751 4752 4753
	u16 alloc_size, start = 0;
	u16 total[I915_MAX_PLANES] = {};
	u16 uv_total[I915_MAX_PLANES] = {};
4754
	u64 total_data_rate;
4755
	enum plane_id plane_id;
4756
	int num_active;
4757 4758
	u64 plane_data_rate[I915_MAX_PLANES] = {};
	u64 uv_plane_data_rate[I915_MAX_PLANES] = {};
4759
	u32 blocks;
4760
	int level;
4761

4762
	/* Clear the partitioning for disabled planes. */
4763 4764
	memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
	memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
4765

4766
	if (!crtc_state->hw.active) {
4767
		alloc->start = alloc->end = 0;
4768 4769 4770
		return 0;
	}

4771 4772
	if (INTEL_GEN(dev_priv) >= 11)
		total_data_rate =
4773
			icl_get_total_relative_data_rate(crtc_state,
4774 4775
							 plane_data_rate);
	else
4776
		total_data_rate =
4777
			skl_get_total_relative_data_rate(crtc_state,
4778 4779
							 plane_data_rate,
							 uv_plane_data_rate);
4780

4781
	skl_ddb_get_pipe_allocation_limits(dev_priv, crtc_state, total_data_rate,
4782
					   alloc, &num_active);
4783
	alloc_size = skl_ddb_entry_size(alloc);
4784
	if (alloc_size == 0)
4785
		return 0;
4786

4787
	/* Allocate fixed number of blocks for cursor. */
4788
	total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
4789
	alloc_size -= total[PLANE_CURSOR];
4790
	crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
4791
		alloc->end - total[PLANE_CURSOR];
4792
	crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
4793 4794 4795

	if (total_data_rate == 0)
		return 0;
4796

4797
	/*
4798 4799
	 * Find the highest watermark level for which we can satisfy the block
	 * requirement of active planes.
4800
	 */
4801
	for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
4802
		blocks = 0;
4803
		for_each_plane_id_on_crtc(crtc, plane_id) {
4804
			const struct skl_plane_wm *wm =
4805
				&crtc_state->wm.skl.optimal.planes[plane_id];
4806 4807

			if (plane_id == PLANE_CURSOR) {
4808
				if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
4809 4810
					drm_WARN_ON(&dev_priv->drm,
						    wm->wm[level].min_ddb_alloc != U16_MAX);
4811 4812 4813
					blocks = U32_MAX;
					break;
				}
4814
				continue;
4815
			}
4816

4817 4818
			blocks += wm->wm[level].min_ddb_alloc;
			blocks += wm->uv_wm[level].min_ddb_alloc;
4819 4820
		}

4821
		if (blocks <= alloc_size) {
4822 4823 4824
			alloc_size -= blocks;
			break;
		}
4825 4826
	}

4827
	if (level < 0) {
4828 4829 4830 4831
		drm_dbg_kms(&dev_priv->drm,
			    "Requested display configuration exceeds system DDB limitations");
		drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
			    blocks, alloc_size);
4832 4833 4834
		return -EINVAL;
	}

4835
	/*
4836 4837 4838
	 * Grant each plane the blocks it requires at the highest achievable
	 * watermark level, plus an extra share of the leftover blocks
	 * proportional to its relative data rate.
4839
	 */
4840
	for_each_plane_id_on_crtc(crtc, plane_id) {
4841
		const struct skl_plane_wm *wm =
4842
			&crtc_state->wm.skl.optimal.planes[plane_id];
4843 4844
		u64 rate;
		u16 extra;
4845

4846
		if (plane_id == PLANE_CURSOR)
4847 4848
			continue;

4849
		/*
4850 4851
		 * We've accounted for all active planes; remaining planes are
		 * all disabled.
4852
		 */
4853 4854
		if (total_data_rate == 0)
			break;
4855

4856 4857 4858 4859
		rate = plane_data_rate[plane_id];
		extra = min_t(u16, alloc_size,
			      DIV64_U64_ROUND_UP(alloc_size * rate,
						 total_data_rate));
4860
		total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
4861 4862
		alloc_size -= extra;
		total_data_rate -= rate;
4863

4864 4865
		if (total_data_rate == 0)
			break;
4866

4867 4868 4869 4870
		rate = uv_plane_data_rate[plane_id];
		extra = min_t(u16, alloc_size,
			      DIV64_U64_ROUND_UP(alloc_size * rate,
						 total_data_rate));
4871
		uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
4872 4873 4874
		alloc_size -= extra;
		total_data_rate -= rate;
	}
4875
	drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
4876 4877 4878

	/* Set the actual DDB start/end points for each plane */
	start = alloc->start;
4879
	for_each_plane_id_on_crtc(crtc, plane_id) {
4880
		struct skl_ddb_entry *plane_alloc =
4881
			&crtc_state->wm.skl.plane_ddb_y[plane_id];
4882
		struct skl_ddb_entry *uv_plane_alloc =
4883
			&crtc_state->wm.skl.plane_ddb_uv[plane_id];
4884 4885 4886 4887

		if (plane_id == PLANE_CURSOR)
			continue;

4888
		/* Gen11+ uses a separate plane for UV watermarks */
4889 4890
		drm_WARN_ON(&dev_priv->drm,
			    INTEL_GEN(dev_priv) >= 11 && uv_total[plane_id]);
4891 4892 4893 4894 4895 4896 4897

		/* Leave disabled planes at (0,0) */
		if (total[plane_id]) {
			plane_alloc->start = start;
			start += total[plane_id];
			plane_alloc->end = start;
		}
4898

4899 4900 4901 4902
		if (uv_total[plane_id]) {
			uv_plane_alloc->start = start;
			start += uv_total[plane_id];
			uv_plane_alloc->end = start;
4903
		}
4904
	}
4905

4906 4907 4908 4909 4910 4911 4912
	/*
	 * When we calculated watermark values we didn't know how high
	 * of a level we'd actually be able to hit, so we just marked
	 * all levels as "enabled."  Go back now and disable the ones
	 * that aren't actually possible.
	 */
	for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
4913
		for_each_plane_id_on_crtc(crtc, plane_id) {
4914
			struct skl_plane_wm *wm =
4915
				&crtc_state->wm.skl.optimal.planes[plane_id];
4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931

			/*
			 * We only disable the watermarks for each plane if
			 * they exceed the ddb allocation of said plane. This
			 * is done so that we don't end up touching cursor
			 * watermarks needlessly when some other plane reduces
			 * our max possible watermark level.
			 *
			 * Bspec has this to say about the PLANE_WM enable bit:
			 * "All the watermarks at this level for all enabled
			 *  planes must be enabled before the level will be used."
			 * So this is actually safe to do.
			 */
			if (wm->wm[level].min_ddb_alloc > total[plane_id] ||
			    wm->uv_wm[level].min_ddb_alloc > uv_total[plane_id])
				memset(&wm->wm[level], 0, sizeof(wm->wm[level]));
4932

4933
			/*
4934
			 * Wa_1408961008:icl, ehl
4935 4936
			 * Underruns with WM1+ disabled
			 */
4937
			if (IS_GEN(dev_priv, 11) &&
4938 4939
			    level == 1 && wm->wm[0].plane_en) {
				wm->wm[level].plane_res_b = wm->wm[0].plane_res_b;
4940 4941
				wm->wm[level].plane_res_l = wm->wm[0].plane_res_l;
				wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
4942
			}
4943 4944 4945 4946 4947 4948 4949
		}
	}

	/*
	 * Go back and disable the transition watermark if it turns out we
	 * don't have enough DDB blocks for it.
	 */
4950
	for_each_plane_id_on_crtc(crtc, plane_id) {
4951
		struct skl_plane_wm *wm =
4952
			&crtc_state->wm.skl.optimal.planes[plane_id];
4953

4954
		if (wm->trans_wm.plane_res_b >= total[plane_id])
4955
			memset(&wm->trans_wm, 0, sizeof(wm->trans_wm));
4956 4957
	}

4958
	return 0;
4959 4960
}

4961 4962
/*
 * The max latency should be 257 (max the punit can code is 255 and we add 2us
4963
 * for the read latency) and cpp should always be <= 8, so that
4964 4965 4966
 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
*/
4967
static uint_fixed_16_16_t
4968 4969
skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
	       u8 cpp, u32 latency, u32 dbuf_block_size)
4970
{
4971
	u32 wm_intermediate_val;
4972
	uint_fixed_16_16_t ret;
4973 4974

	if (latency == 0)
4975
		return FP_16_16_MAX;
4976

4977
	wm_intermediate_val = latency * pixel_rate * cpp;
4978
	ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
4979

4980
	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
4981 4982
		ret = add_fixed16_u32(ret, 1);

4983 4984 4985
	return ret;
}

4986 4987 4988
static uint_fixed_16_16_t
skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
	       uint_fixed_16_16_t plane_blocks_per_line)
4989
{
4990
	u32 wm_intermediate_val;
4991
	uint_fixed_16_16_t ret;
4992 4993

	if (latency == 0)
4994
		return FP_16_16_MAX;
4995 4996

	wm_intermediate_val = latency * pixel_rate;
4997 4998
	wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
					   pipe_htotal * 1000);
4999
	ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5000 5001 5002
	return ret;
}

5003
static uint_fixed_16_16_t
5004
intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5005
{
5006 5007
	u32 pixel_rate;
	u32 crtc_htotal;
5008 5009
	uint_fixed_16_16_t linetime_us;

5010
	if (!crtc_state->hw.active)
5011
		return u32_to_fixed16(0);
5012

5013
	pixel_rate = crtc_state->pixel_rate;
5014 5015

	if (WARN_ON(pixel_rate == 0))
5016
		return u32_to_fixed16(0);
5017

5018
	crtc_htotal = crtc_state->hw.adjusted_mode.crtc_htotal;
5019
	linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5020 5021 5022 5023

	return linetime_us;
}

5024
static u32
5025 5026
skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *crtc_state,
			      const struct intel_plane_state *plane_state)
5027
{
5028
	u64 adjusted_pixel_rate;
5029
	uint_fixed_16_16_t downscale_amount;
5030 5031

	/* Shouldn't reach here on disabled planes... */
5032
	if (WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state)))
5033 5034 5035 5036 5037 5038
		return 0;

	/*
	 * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
	 * with additional adjustments for plane-specific scaling.
	 */
5039 5040
	adjusted_pixel_rate = crtc_state->pixel_rate;
	downscale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
5041

5042 5043
	return mul_round_up_u32_fixed16(adjusted_pixel_rate,
					    downscale_amount);
5044 5045
}

5046
static int
5047 5048 5049 5050 5051
skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
		      int width, const struct drm_format_info *format,
		      u64 modifier, unsigned int rotation,
		      u32 plane_pixel_rate, struct skl_wm_params *wp,
		      int color_plane)
5052
{
5053
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5054
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5055
	u32 interm_pbpl;
5056

5057
	/* only planar format has two planes */
5058 5059
	if (color_plane == 1 &&
	    !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5060 5061
		drm_dbg_kms(&dev_priv->drm,
			    "Non planar format have single plane\n");
5062 5063 5064
		return -EINVAL;
	}

5065 5066 5067 5068 5069 5070 5071
	wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
		      modifier == I915_FORMAT_MOD_Yf_TILED ||
		      modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
		      modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
	wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
	wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
			 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5072
	wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5073

5074
	wp->width = width;
5075
	if (color_plane == 1 && wp->is_planar)
5076 5077
		wp->width /= 2;

5078 5079
	wp->cpp = format->cpp[color_plane];
	wp->plane_pixel_rate = plane_pixel_rate;
5080

5081
	if (INTEL_GEN(dev_priv) >= 11 &&
5082
	    modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
5083 5084 5085 5086
		wp->dbuf_block_size = 256;
	else
		wp->dbuf_block_size = 512;

5087
	if (drm_rotation_90_or_270(rotation)) {
5088
		switch (wp->cpp) {
5089
		case 1:
5090
			wp->y_min_scanlines = 16;
5091 5092
			break;
		case 2:
5093
			wp->y_min_scanlines = 8;
5094 5095
			break;
		case 4:
5096
			wp->y_min_scanlines = 4;
5097
			break;
5098
		default:
5099
			MISSING_CASE(wp->cpp);
5100
			return -EINVAL;
5101 5102
		}
	} else {
5103
		wp->y_min_scanlines = 4;
5104 5105
	}

5106
	if (skl_needs_memory_bw_wa(dev_priv))
5107
		wp->y_min_scanlines *= 2;
5108

5109 5110 5111
	wp->plane_bytes_per_line = wp->width * wp->cpp;
	if (wp->y_tiled) {
		interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5112 5113
					   wp->y_min_scanlines,
					   wp->dbuf_block_size);
5114

5115
		if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5116 5117
			interm_pbpl++;

5118 5119
		wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
							wp->y_min_scanlines);
5120
	} else {
5121
		interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5122 5123 5124 5125 5126 5127
					   wp->dbuf_block_size);

		if (!wp->x_tiled ||
		    INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
			interm_pbpl++;

5128
		wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5129 5130
	}

5131 5132
	wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
					     wp->plane_blocks_per_line);
5133

5134
	wp->linetime_us = fixed16_to_u32_round_up(
5135
					intel_get_linetime_us(crtc_state));
5136 5137 5138 5139

	return 0;
}

5140 5141 5142 5143 5144
static int
skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
			    const struct intel_plane_state *plane_state,
			    struct skl_wm_params *wp, int color_plane)
{
5145
	const struct drm_framebuffer *fb = plane_state->hw.fb;
5146 5147
	int width;

5148 5149 5150 5151 5152
	/*
	 * Src coordinates are already rotated by 270 degrees for
	 * the 90/270 degree plane rotation cases (to match the
	 * GTT mapping), hence no need to account for rotation here.
	 */
5153
	width = drm_rect_width(&plane_state->uapi.src) >> 16;
5154 5155 5156

	return skl_compute_wm_params(crtc_state, width,
				     fb->format, fb->modifier,
5157
				     plane_state->hw.rotation,
5158 5159 5160 5161
				     skl_adjusted_plane_pixel_rate(crtc_state, plane_state),
				     wp, color_plane);
}

5162 5163 5164 5165 5166 5167 5168 5169 5170
static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
{
	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
		return true;

	/* The number of lines are ignored for the level 0 watermark. */
	return level > 0;
}

5171
static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5172
				 int level,
5173
				 unsigned int latency,
5174 5175 5176
				 const struct skl_wm_params *wp,
				 const struct skl_wm_level *result_prev,
				 struct skl_wm_level *result /* out */)
5177
{
5178
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5179 5180
	uint_fixed_16_16_t method1, method2;
	uint_fixed_16_16_t selected_result;
5181
	u32 res_blocks, res_lines, min_ddb_alloc = 0;
5182

5183 5184 5185
	if (latency == 0) {
		/* reject it */
		result->min_ddb_alloc = U16_MAX;
5186
		return;
5187
	}
5188

5189 5190 5191 5192
	/*
	 * WaIncreaseLatencyIPCEnabled: kbl,cfl
	 * Display WA #1141: kbl,cfl
	 */
5193
	if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) &&
5194 5195 5196
	    dev_priv->ipc_enabled)
		latency += 4;

5197
	if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5198 5199 5200
		latency += 15;

	method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5201
				 wp->cpp, latency, wp->dbuf_block_size);
5202
	method2 = skl_wm_method2(wp->plane_pixel_rate,
5203
				 crtc_state->hw.adjusted_mode.crtc_htotal,
5204
				 latency,
5205
				 wp->plane_blocks_per_line);
5206

5207 5208
	if (wp->y_tiled) {
		selected_result = max_fixed16(method2, wp->y_tile_minimum);
5209
	} else {
5210
		if ((wp->cpp * crtc_state->hw.adjusted_mode.crtc_htotal /
5211
		     wp->dbuf_block_size < 1) &&
5212
		     (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5213
			selected_result = method2;
5214
		} else if (latency >= wp->linetime_us) {
5215
			if (IS_GEN(dev_priv, 9) &&
5216 5217 5218 5219 5220
			    !IS_GEMINILAKE(dev_priv))
				selected_result = min_fixed16(method1, method2);
			else
				selected_result = method2;
		} else {
5221
			selected_result = method1;
5222
		}
5223
	}
5224

5225
	res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
5226
	res_lines = div_round_up_fixed16(selected_result,
5227
					 wp->plane_blocks_per_line);
5228

5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243
	if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) {
		/* Display WA #1125: skl,bxt,kbl */
		if (level == 0 && wp->rc_surface)
			res_blocks +=
				fixed16_to_u32_round_up(wp->y_tile_minimum);

		/* Display WA #1126: skl,bxt,kbl */
		if (level >= 1 && level <= 7) {
			if (wp->y_tiled) {
				res_blocks +=
				    fixed16_to_u32_round_up(wp->y_tile_minimum);
				res_lines += wp->y_min_scanlines;
			} else {
				res_blocks++;
			}
5244

5245 5246 5247 5248 5249 5250 5251 5252 5253
			/*
			 * Make sure result blocks for higher latency levels are
			 * atleast as high as level below the current level.
			 * Assumption in DDB algorithm optimization for special
			 * cases. Also covers Display WA #1125 for RC.
			 */
			if (result_prev->plane_res_b > res_blocks)
				res_blocks = result_prev->plane_res_b;
		}
5254
	}
5255

5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273
	if (INTEL_GEN(dev_priv) >= 11) {
		if (wp->y_tiled) {
			int extra_lines;

			if (res_lines % wp->y_min_scanlines == 0)
				extra_lines = wp->y_min_scanlines;
			else
				extra_lines = wp->y_min_scanlines * 2 -
					res_lines % wp->y_min_scanlines;

			min_ddb_alloc = mul_round_up_u32_fixed16(res_lines + extra_lines,
								 wp->plane_blocks_per_line);
		} else {
			min_ddb_alloc = res_blocks +
				DIV_ROUND_UP(res_blocks, 10);
		}
	}

5274 5275 5276
	if (!skl_wm_has_lines(dev_priv, level))
		res_lines = 0;

5277 5278 5279
	if (res_lines > 31) {
		/* reject it */
		result->min_ddb_alloc = U16_MAX;
5280
		return;
5281
	}
5282 5283 5284 5285 5286 5287 5288

	/*
	 * If res_lines is valid, assume we can use this watermark level
	 * for now.  We'll come back and disable it after we calculate the
	 * DDB allocation if it turns out we don't actually have enough
	 * blocks to satisfy it.
	 */
5289 5290
	result->plane_res_b = res_blocks;
	result->plane_res_l = res_lines;
5291 5292
	/* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
	result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1;
5293
	result->plane_en = true;
5294 5295
}

5296
static void
5297
skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5298
		      const struct skl_wm_params *wm_params,
5299
		      struct skl_wm_level *levels)
5300
{
5301
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5302
	int level, max_level = ilk_wm_max_level(dev_priv);
5303
	struct skl_wm_level *result_prev = &levels[0];
5304

5305
	for (level = 0; level <= max_level; level++) {
5306
		struct skl_wm_level *result = &levels[level];
5307
		unsigned int latency = dev_priv->wm.skl_latency[level];
5308

5309 5310
		skl_compute_plane_wm(crtc_state, level, latency,
				     wm_params, result_prev, result);
5311 5312

		result_prev = result;
5313
	}
5314 5315
}

5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329
static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
				const struct skl_wm_params *wm_params,
				struct skl_plane_wm *plane_wm)
{
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
	struct skl_wm_level *sagv_wm = &plane_wm->sagv_wm0;
	struct skl_wm_level *levels = plane_wm->wm;
	unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;

	skl_compute_plane_wm(crtc_state, 0, latency,
			     wm_params, &levels[0],
			     sagv_wm);
}

5330
static void skl_compute_transition_wm(const struct intel_crtc_state *crtc_state,
5331
				      const struct skl_wm_params *wp,
5332
				      struct skl_plane_wm *wm)
5333
{
5334
	struct drm_device *dev = crtc_state->uapi.crtc->dev;
5335
	const struct drm_i915_private *dev_priv = to_i915(dev);
5336
	u16 trans_min, trans_amount, trans_y_tile_min;
5337
	u16 wm0_sel_res_b, trans_offset_b, res_blocks;
5338 5339 5340

	/* Transition WM don't make any sense if ipc is disabled */
	if (!dev_priv->ipc_enabled)
5341
		return;
5342

5343 5344 5345 5346 5347 5348 5349
	/*
	 * WaDisableTWM:skl,kbl,cfl,bxt
	 * Transition WM are not recommended by HW team for GEN9
	 */
	if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
		return;

5350
	if (INTEL_GEN(dev_priv) >= 11)
5351
		trans_min = 4;
5352 5353 5354 5355 5356 5357 5358 5359
	else
		trans_min = 14;

	/* Display WA #1140: glk,cnl */
	if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
		trans_amount = 0;
	else
		trans_amount = 10; /* This is configurable amount */
5360 5361 5362

	trans_offset_b = trans_min + trans_amount;

5363 5364 5365 5366 5367 5368 5369 5370 5371 5372
	/*
	 * The spec asks for Selected Result Blocks for wm0 (the real value),
	 * not Result Blocks (the integer value). Pay attention to the capital
	 * letters. The value wm_l0->plane_res_b is actually Result Blocks, but
	 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
	 * and since we later will have to get the ceiling of the sum in the
	 * transition watermarks calculation, we can just pretend Selected
	 * Result Blocks is Result Blocks minus 1 and it should work for the
	 * current platforms.
	 */
5373
	wm0_sel_res_b = wm->wm[0].plane_res_b - 1;
5374

5375
	if (wp->y_tiled) {
5376 5377
		trans_y_tile_min =
			(u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5378
		res_blocks = max(wm0_sel_res_b, trans_y_tile_min) +
5379 5380
				trans_offset_b;
	} else {
5381
		res_blocks = wm0_sel_res_b + trans_offset_b;
5382 5383
	}

5384 5385 5386 5387 5388 5389 5390
	/*
	 * Just assume we can enable the transition watermark.  After
	 * computing the DDB we'll come back and disable it if that
	 * assumption turns out to be false.
	 */
	wm->trans_wm.plane_res_b = res_blocks + 1;
	wm->trans_wm.plane_en = true;
5391 5392
}

5393
static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5394 5395
				     const struct intel_plane_state *plane_state,
				     enum plane_id plane_id, int color_plane)
5396
{
5397 5398
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5399
	struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5400 5401 5402
	struct skl_wm_params wm_params;
	int ret;

5403
	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5404 5405 5406 5407
					  &wm_params, color_plane);
	if (ret)
		return ret;

5408
	skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5409 5410 5411 5412

	if (INTEL_GEN(dev_priv) >= 12)
		tgl_compute_sagv_wm(crtc_state, &wm_params, wm);

5413
	skl_compute_transition_wm(crtc_state, &wm_params, wm);
5414 5415 5416 5417

	return 0;
}

5418
static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5419 5420
				 const struct intel_plane_state *plane_state,
				 enum plane_id plane_id)
5421
{
5422
	struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5423 5424 5425
	struct skl_wm_params wm_params;
	int ret;

5426
	wm->is_planar = true;
5427 5428

	/* uv plane watermarks must also be validated for NV12/Planar */
5429
	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5430 5431 5432
					  &wm_params, 1);
	if (ret)
		return ret;
5433

5434
	skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5435

5436
	return 0;
5437 5438
}

5439
static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5440
			      const struct intel_plane_state *plane_state)
5441
{
5442
	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5443
	const struct drm_framebuffer *fb = plane_state->hw.fb;
5444
	enum plane_id plane_id = plane->id;
5445 5446
	int ret;

5447 5448 5449
	if (!intel_wm_plane_visible(crtc_state, plane_state))
		return 0;

5450
	ret = skl_build_plane_wm_single(crtc_state, plane_state,
5451
					plane_id, 0);
5452 5453 5454
	if (ret)
		return ret;

5455
	if (fb->format->is_yuv && fb->format->num_planes > 1) {
5456
		ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5457 5458 5459 5460 5461 5462 5463 5464
					    plane_id);
		if (ret)
			return ret;
	}

	return 0;
}

5465
static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5466 5467
			      const struct intel_plane_state *plane_state)
{
5468
	enum plane_id plane_id = to_intel_plane(plane_state->uapi.plane)->id;
5469 5470 5471
	int ret;

	/* Watermarks calculated in master */
5472
	if (plane_state->planar_slave)
5473 5474
		return 0;

5475
	if (plane_state->planar_linked_plane) {
5476
		const struct drm_framebuffer *fb = plane_state->hw.fb;
5477
		enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
5478 5479 5480 5481 5482

		WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state));
		WARN_ON(!fb->format->is_yuv ||
			fb->format->num_planes == 1);

5483
		ret = skl_build_plane_wm_single(crtc_state, plane_state,
5484 5485 5486 5487
						y_plane_id, 0);
		if (ret)
			return ret;

5488
		ret = skl_build_plane_wm_single(crtc_state, plane_state,
5489 5490 5491 5492
						plane_id, 1);
		if (ret)
			return ret;
	} else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5493
		ret = skl_build_plane_wm_single(crtc_state, plane_state,
5494 5495 5496 5497 5498 5499
						plane_id, 0);
		if (ret)
			return ret;
	}

	return 0;
5500 5501
}

5502
static int skl_build_pipe_wm(struct intel_crtc_state *crtc_state)
5503
{
5504
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5505
	struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5506 5507
	struct intel_plane *plane;
	const struct intel_plane_state *plane_state;
5508
	int ret;
5509

5510 5511 5512 5513 5514 5515
	/*
	 * We'll only calculate watermarks for planes that are actually
	 * enabled, so make sure all other planes are set as disabled.
	 */
	memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));

5516 5517
	intel_atomic_crtc_state_for_each_plane_state(plane, plane_state,
						     crtc_state) {
5518

5519
		if (INTEL_GEN(dev_priv) >= 11)
5520
			ret = icl_build_plane_wm(crtc_state, plane_state);
5521
		else
5522
			ret = skl_build_plane_wm(crtc_state, plane_state);
5523 5524
		if (ret)
			return ret;
5525
	}
5526

5527
	return 0;
5528 5529
}

5530 5531
static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
				i915_reg_t reg,
5532 5533 5534
				const struct skl_ddb_entry *entry)
{
	if (entry->end)
5535 5536
		intel_de_write_fw(dev_priv, reg,
				  (entry->end - 1) << 16 | entry->start);
5537
	else
5538
		intel_de_write_fw(dev_priv, reg, 0);
5539 5540
}

5541 5542 5543 5544
static void skl_write_wm_level(struct drm_i915_private *dev_priv,
			       i915_reg_t reg,
			       const struct skl_wm_level *level)
{
5545
	u32 val = 0;
5546

5547
	if (level->plane_en)
5548
		val |= PLANE_WM_EN;
5549 5550 5551 5552
	if (level->ignore_lines)
		val |= PLANE_WM_IGNORE_LINES;
	val |= level->plane_res_b;
	val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
5553

5554
	intel_de_write_fw(dev_priv, reg, val);
5555 5556
}

5557 5558
void skl_write_plane_wm(struct intel_plane *plane,
			const struct intel_crtc_state *crtc_state)
5559
{
5560
	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5561
	int level, max_level = ilk_wm_max_level(dev_priv);
5562 5563 5564 5565 5566 5567 5568 5569
	enum plane_id plane_id = plane->id;
	enum pipe pipe = plane->pipe;
	const struct skl_plane_wm *wm =
		&crtc_state->wm.skl.optimal.planes[plane_id];
	const struct skl_ddb_entry *ddb_y =
		&crtc_state->wm.skl.plane_ddb_y[plane_id];
	const struct skl_ddb_entry *ddb_uv =
		&crtc_state->wm.skl.plane_ddb_uv[plane_id];
5570 5571

	for (level = 0; level <= max_level; level++) {
5572 5573 5574 5575
		const struct skl_wm_level *wm_level;

		wm_level = skl_plane_wm_level(crtc_state, plane_id, level);

5576
		skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5577
				   wm_level);
5578
	}
5579
	skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5580
			   &wm->trans_wm);
5581

5582
	if (INTEL_GEN(dev_priv) >= 11) {
5583
		skl_ddb_entry_write(dev_priv,
5584 5585
				    PLANE_BUF_CFG(pipe, plane_id), ddb_y);
		return;
5586
	}
5587 5588 5589 5590 5591 5592 5593 5594

	if (wm->is_planar)
		swap(ddb_y, ddb_uv);

	skl_ddb_entry_write(dev_priv,
			    PLANE_BUF_CFG(pipe, plane_id), ddb_y);
	skl_ddb_entry_write(dev_priv,
			    PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5595 5596
}

5597 5598
void skl_write_cursor_wm(struct intel_plane *plane,
			 const struct intel_crtc_state *crtc_state)
5599
{
5600
	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5601
	int level, max_level = ilk_wm_max_level(dev_priv);
5602 5603 5604 5605 5606 5607
	enum plane_id plane_id = plane->id;
	enum pipe pipe = plane->pipe;
	const struct skl_plane_wm *wm =
		&crtc_state->wm.skl.optimal.planes[plane_id];
	const struct skl_ddb_entry *ddb =
		&crtc_state->wm.skl.plane_ddb_y[plane_id];
5608 5609

	for (level = 0; level <= max_level; level++) {
5610 5611 5612 5613
		const struct skl_wm_level *wm_level;

		wm_level = skl_plane_wm_level(crtc_state, plane_id, level);

5614
		skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5615
				   wm_level);
5616
	}
5617
	skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
5618

5619
	skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5620 5621
}

5622 5623 5624
bool skl_wm_level_equals(const struct skl_wm_level *l1,
			 const struct skl_wm_level *l2)
{
5625
	return l1->plane_en == l2->plane_en &&
5626
		l1->ignore_lines == l2->ignore_lines &&
5627 5628 5629
		l1->plane_res_l == l2->plane_res_l &&
		l1->plane_res_b == l2->plane_res_b;
}
5630

5631 5632 5633 5634 5635
static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
				const struct skl_plane_wm *wm1,
				const struct skl_plane_wm *wm2)
{
	int level, max_level = ilk_wm_max_level(dev_priv);
5636

5637
	for (level = 0; level <= max_level; level++) {
5638 5639 5640 5641 5642 5643
		/*
		 * We don't check uv_wm as the hardware doesn't actually
		 * use it. It only gets used for calculating the required
		 * ddb allocation.
		 */
		if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5644 5645 5646 5647
			return false;
	}

	return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm);
5648 5649
}

5650 5651
static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
				    const struct skl_ddb_entry *b)
5652
{
5653
	return a->start < b->end && b->start < a->end;
5654 5655
}

5656
bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5657
				 const struct skl_ddb_entry *entries,
5658
				 int num_entries, int ignore_idx)
5659
{
5660
	int i;
5661

5662 5663 5664
	for (i = 0; i < num_entries; i++) {
		if (i != ignore_idx &&
		    skl_ddb_entries_overlap(ddb, &entries[i]))
5665
			return true;
5666
	}
5667

5668
	return false;
5669 5670
}

5671
static int
5672 5673
skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
			    struct intel_crtc_state *new_crtc_state)
5674
{
5675 5676
	struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
5677 5678
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	struct intel_plane *plane;
5679

5680 5681 5682
	for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
		struct intel_plane_state *plane_state;
		enum plane_id plane_id = plane->id;
5683

5684 5685 5686 5687
		if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
					&new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
		    skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
					&new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
5688 5689
			continue;

5690
		plane_state = intel_atomic_get_plane_state(state, plane);
5691 5692
		if (IS_ERR(plane_state))
			return PTR_ERR(plane_state);
5693

5694
		new_crtc_state->update_planes |= BIT(plane_id);
5695 5696 5697 5698 5699 5700
	}

	return 0;
}

static int
5701
skl_compute_ddb(struct intel_atomic_state *state)
5702
{
5703
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5704 5705
	struct intel_crtc_state *old_crtc_state;
	struct intel_crtc_state *new_crtc_state;
5706 5707
	struct intel_crtc *crtc;
	int ret, i;
5708

5709
	state->enabled_dbuf_slices_mask = dev_priv->enabled_dbuf_slices_mask;
5710

5711
	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5712
					    new_crtc_state, i) {
5713
		ret = skl_allocate_pipe_ddb(new_crtc_state);
5714 5715 5716
		if (ret)
			return ret;

5717 5718
		ret = skl_ddb_add_affected_planes(old_crtc_state,
						  new_crtc_state);
5719 5720
		if (ret)
			return ret;
5721 5722 5723 5724 5725
	}

	return 0;
}

5726 5727 5728 5729 5730
static char enast(bool enable)
{
	return enable ? '*' : ' ';
}

5731
static void
5732
skl_print_wm_changes(struct intel_atomic_state *state)
5733
{
5734 5735 5736 5737 5738
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
	const struct intel_crtc_state *old_crtc_state;
	const struct intel_crtc_state *new_crtc_state;
	struct intel_plane *plane;
	struct intel_crtc *crtc;
5739
	int i;
5740

5741
	if (!drm_debug_enabled(DRM_UT_KMS))
5742 5743
		return;

5744 5745
	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
					    new_crtc_state, i) {
5746 5747 5748 5749 5750
		const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;

		old_pipe_wm = &old_crtc_state->wm.skl.optimal;
		new_pipe_wm = &new_crtc_state->wm.skl.optimal;

5751 5752
		for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
			enum plane_id plane_id = plane->id;
5753 5754
			const struct skl_ddb_entry *old, *new;

5755 5756
			old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
			new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
5757 5758 5759 5760

			if (skl_ddb_entry_equal(old, new))
				continue;

5761 5762 5763 5764 5765
			drm_dbg_kms(&dev_priv->drm,
				    "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
				    plane->base.base.id, plane->base.name,
				    old->start, old->end, new->start, new->end,
				    skl_ddb_entry_size(old), skl_ddb_entry_size(new));
5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777
		}

		for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
			enum plane_id plane_id = plane->id;
			const struct skl_plane_wm *old_wm, *new_wm;

			old_wm = &old_pipe_wm->planes[plane_id];
			new_wm = &new_pipe_wm->planes[plane_id];

			if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
				continue;

5778
			drm_dbg_kms(&dev_priv->drm,
5779 5780
				    "[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm"
				    " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm\n",
5781 5782 5783 5784 5785 5786
				    plane->base.base.id, plane->base.name,
				    enast(old_wm->wm[0].plane_en), enast(old_wm->wm[1].plane_en),
				    enast(old_wm->wm[2].plane_en), enast(old_wm->wm[3].plane_en),
				    enast(old_wm->wm[4].plane_en), enast(old_wm->wm[5].plane_en),
				    enast(old_wm->wm[6].plane_en), enast(old_wm->wm[7].plane_en),
				    enast(old_wm->trans_wm.plane_en),
5787
				    enast(old_wm->sagv_wm0.plane_en),
5788 5789 5790 5791
				    enast(new_wm->wm[0].plane_en), enast(new_wm->wm[1].plane_en),
				    enast(new_wm->wm[2].plane_en), enast(new_wm->wm[3].plane_en),
				    enast(new_wm->wm[4].plane_en), enast(new_wm->wm[5].plane_en),
				    enast(new_wm->wm[6].plane_en), enast(new_wm->wm[7].plane_en),
5792 5793
				    enast(new_wm->trans_wm.plane_en),
				    enast(new_wm->sagv_wm0.plane_en));
5794 5795

			drm_dbg_kms(&dev_priv->drm,
5796 5797
				    "[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d"
				      " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d\n",
5798 5799 5800 5801 5802 5803 5804 5805 5806 5807
				    plane->base.base.id, plane->base.name,
				    enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].plane_res_l,
				    enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].plane_res_l,
				    enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].plane_res_l,
				    enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].plane_res_l,
				    enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].plane_res_l,
				    enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].plane_res_l,
				    enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].plane_res_l,
				    enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].plane_res_l,
				    enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.plane_res_l,
5808
				    enast(old_wm->sagv_wm0.ignore_lines), old_wm->sagv_wm0.plane_res_l,
5809 5810 5811 5812 5813 5814 5815 5816 5817

				    enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].plane_res_l,
				    enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].plane_res_l,
				    enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].plane_res_l,
				    enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].plane_res_l,
				    enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].plane_res_l,
				    enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].plane_res_l,
				    enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].plane_res_l,
				    enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].plane_res_l,
5818 5819
				    enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.plane_res_l,
				    enast(new_wm->sagv_wm0.ignore_lines), new_wm->sagv_wm0.plane_res_l);
5820 5821

			drm_dbg_kms(&dev_priv->drm,
5822 5823
				    "[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
				    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5824 5825 5826 5827 5828 5829
				    plane->base.base.id, plane->base.name,
				    old_wm->wm[0].plane_res_b, old_wm->wm[1].plane_res_b,
				    old_wm->wm[2].plane_res_b, old_wm->wm[3].plane_res_b,
				    old_wm->wm[4].plane_res_b, old_wm->wm[5].plane_res_b,
				    old_wm->wm[6].plane_res_b, old_wm->wm[7].plane_res_b,
				    old_wm->trans_wm.plane_res_b,
5830
				    old_wm->sagv_wm0.plane_res_b,
5831 5832 5833 5834
				    new_wm->wm[0].plane_res_b, new_wm->wm[1].plane_res_b,
				    new_wm->wm[2].plane_res_b, new_wm->wm[3].plane_res_b,
				    new_wm->wm[4].plane_res_b, new_wm->wm[5].plane_res_b,
				    new_wm->wm[6].plane_res_b, new_wm->wm[7].plane_res_b,
5835 5836
				    new_wm->trans_wm.plane_res_b,
				    new_wm->sagv_wm0.plane_res_b);
5837 5838

			drm_dbg_kms(&dev_priv->drm,
5839 5840
				    "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
				    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5841 5842 5843 5844 5845 5846
				    plane->base.base.id, plane->base.name,
				    old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
				    old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
				    old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
				    old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
				    old_wm->trans_wm.min_ddb_alloc,
5847
				    old_wm->sagv_wm0.min_ddb_alloc,
5848 5849 5850 5851
				    new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
				    new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
				    new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
				    new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
5852 5853
				    new_wm->trans_wm.min_ddb_alloc,
				    new_wm->sagv_wm0.min_ddb_alloc);
5854 5855 5856 5857
		}
	}
}

5858 5859 5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873
static int intel_add_all_pipes(struct intel_atomic_state *state)
{
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
	struct intel_crtc *crtc;

	for_each_intel_crtc(&dev_priv->drm, crtc) {
		struct intel_crtc_state *crtc_state;

		crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
		if (IS_ERR(crtc_state))
			return PTR_ERR(crtc_state);
	}

	return 0;
}

5874
static int
5875
skl_ddb_add_affected_pipes(struct intel_atomic_state *state)
5876
{
5877
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5878
	int ret;
5879

5880 5881 5882 5883 5884 5885 5886
	/*
	 * If this is our first atomic update following hardware readout,
	 * we can't trust the DDB that the BIOS programmed for us.  Let's
	 * pretend that all pipes switched active status so that we'll
	 * ensure a full DDB recompute.
	 */
	if (dev_priv->wm.distrust_bios_wm) {
5887
		ret = drm_modeset_lock(&dev_priv->drm.mode_config.connection_mutex,
5888
				       state->base.acquire_ctx);
5889 5890 5891
		if (ret)
			return ret;

5892
		state->active_pipe_changes = INTEL_INFO(dev_priv)->pipe_mask;
5893 5894

		/*
5895
		 * We usually only initialize state->active_pipes if we
5896 5897 5898 5899
		 * we're doing a modeset; make sure this field is always
		 * initialized during the sanitization process that happens
		 * on the first commit too.
		 */
5900
		if (!state->modeset)
5901
			state->active_pipes = dev_priv->active_pipes;
5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916
	}

	/*
	 * If the modeset changes which CRTC's are active, we need to
	 * recompute the DDB allocation for *all* active pipes, even
	 * those that weren't otherwise being modified in any way by this
	 * atomic commit.  Due to the shrinking of the per-pipe allocations
	 * when new active CRTC's are added, it's possible for a pipe that
	 * we were already using and aren't changing at all here to suddenly
	 * become invalid if its DDB needs exceeds its new allocation.
	 *
	 * Note that if we wind up doing a full DDB recompute, we can't let
	 * any other display updates race with this transaction, so we need
	 * to grab the lock on *all* CRTC's.
	 */
5917
	if (state->active_pipe_changes || state->modeset) {
5918 5919 5920
		ret = intel_add_all_pipes(state);
		if (ret)
			return ret;
5921 5922 5923 5924 5925
	}

	return 0;
}

5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969
/*
 * To make sure the cursor watermark registers are always consistent
 * with our computed state the following scenario needs special
 * treatment:
 *
 * 1. enable cursor
 * 2. move cursor entirely offscreen
 * 3. disable cursor
 *
 * Step 2. does call .disable_plane() but does not zero the watermarks
 * (since we consider an offscreen cursor still active for the purposes
 * of watermarks). Step 3. would not normally call .disable_plane()
 * because the actual plane visibility isn't changing, and we don't
 * deallocate the cursor ddb until the pipe gets disabled. So we must
 * force step 3. to call .disable_plane() to update the watermark
 * registers properly.
 *
 * Other planes do not suffer from this issues as their watermarks are
 * calculated based on the actual plane visibility. The only time this
 * can trigger for the other planes is during the initial readout as the
 * default value of the watermarks registers is not zero.
 */
static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
				      struct intel_crtc *crtc)
{
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	const struct intel_crtc_state *old_crtc_state =
		intel_atomic_get_old_crtc_state(state, crtc);
	struct intel_crtc_state *new_crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
	struct intel_plane *plane;

	for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
		struct intel_plane_state *plane_state;
		enum plane_id plane_id = plane->id;

		/*
		 * Force a full wm update for every plane on modeset.
		 * Required because the reset value of the wm registers
		 * is non-zero, whereas we want all disabled planes to
		 * have zero watermarks. So if we turn off the relevant
		 * power well the hardware state will go out of sync
		 * with the software state.
		 */
5970
		if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985
		    skl_plane_wm_equals(dev_priv,
					&old_crtc_state->wm.skl.optimal.planes[plane_id],
					&new_crtc_state->wm.skl.optimal.planes[plane_id]))
			continue;

		plane_state = intel_atomic_get_plane_state(state, plane);
		if (IS_ERR(plane_state))
			return PTR_ERR(plane_state);

		new_crtc_state->update_planes |= BIT(plane_id);
	}

	return 0;
}

5986
static int
5987
skl_compute_wm(struct intel_atomic_state *state)
5988
{
5989
	struct intel_crtc *crtc;
5990
	struct intel_crtc_state *new_crtc_state;
5991
	struct intel_crtc_state *old_crtc_state;
5992 5993
	int ret, i;

5994 5995
	ret = skl_ddb_add_affected_pipes(state);
	if (ret)
5996 5997
		return ret;

5998 5999
	/*
	 * Calculate WM's for all pipes that are part of this transaction.
6000
	 * Note that skl_ddb_add_affected_pipes may have added more CRTC's that
6001
	 * weren't otherwise being modified if pipe allocations had to change.
6002
	 */
6003
	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6004 6005
					    new_crtc_state, i) {
		ret = skl_build_pipe_wm(new_crtc_state);
6006 6007
		if (ret)
			return ret;
6008 6009
	}

6010 6011 6012 6013
	ret = skl_compute_ddb(state);
	if (ret)
		return ret;

6014 6015 6016
	ret = intel_compute_sagv_mask(state);
	if (ret)
		return ret;
6017

6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029
	/*
	 * skl_compute_ddb() will have adjusted the final watermarks
	 * based on how much ddb is available. Now we can actually
	 * check if the final watermarks changed.
	 */
	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
					    new_crtc_state, i) {
		ret = skl_wm_add_affected_planes(state, crtc);
		if (ret)
			return ret;
	}

6030
	skl_print_wm_changes(state);
6031

6032 6033 6034
	return 0;
}

6035
static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6036 6037 6038 6039 6040
				  struct intel_wm_config *config)
{
	struct intel_crtc *crtc;

	/* Compute the currently _active_ config */
6041
	for_each_intel_crtc(&dev_priv->drm, crtc) {
6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052
		const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;

		if (!wm->pipe_enabled)
			continue;

		config->sprites_enabled |= wm->sprites_enabled;
		config->sprites_scaled |= wm->sprites_scaled;
		config->num_pipes_active++;
	}
}

6053
static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6054
{
6055
	struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6056
	struct ilk_wm_maximums max;
6057
	struct intel_wm_config config = {};
6058
	struct ilk_wm_values results = {};
6059
	enum intel_ddb_partitioning partitioning;
6060

6061
	ilk_compute_wm_config(dev_priv, &config);
6062

6063 6064
	ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
	ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6065 6066

	/* 5/6 split only in single pipe config on IVB+ */
6067
	if (INTEL_GEN(dev_priv) >= 7 &&
6068
	    config.num_pipes_active == 1 && config.sprites_enabled) {
6069 6070
		ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
		ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6071

6072
		best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6073
	} else {
6074
		best_lp_wm = &lp_wm_1_2;
6075 6076
	}

6077
	partitioning = (best_lp_wm == &lp_wm_1_2) ?
6078
		       INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6079

6080
	ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6081

6082
	ilk_write_wm_values(dev_priv, &results);
6083 6084
}

6085
static void ilk_initial_watermarks(struct intel_atomic_state *state,
6086
				   struct intel_crtc *crtc)
6087
{
6088 6089 6090
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	const struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
6091

6092
	mutex_lock(&dev_priv->wm.wm_mutex);
6093
	crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6094 6095 6096
	ilk_program_watermarks(dev_priv);
	mutex_unlock(&dev_priv->wm.wm_mutex);
}
6097

6098
static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6099
				    struct intel_crtc *crtc)
6100
{
6101 6102 6103
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	const struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
6104 6105 6106

	if (!crtc_state->wm.need_postvbl_update)
		return;
6107

6108
	mutex_lock(&dev_priv->wm.wm_mutex);
6109 6110
	crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
	ilk_program_watermarks(dev_priv);
6111
	mutex_unlock(&dev_priv->wm.wm_mutex);
6112 6113
}

6114
static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6115
{
6116
	level->plane_en = val & PLANE_WM_EN;
6117
	level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6118 6119 6120
	level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
	level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
		PLANE_WM_LINES_MASK;
6121 6122
}

6123
void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6124
			      struct skl_pipe_wm *out)
6125
{
6126 6127
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	enum pipe pipe = crtc->pipe;
6128 6129
	int level, max_level;
	enum plane_id plane_id;
6130
	u32 val;
6131

6132
	max_level = ilk_wm_max_level(dev_priv);
6133

6134
	for_each_plane_id_on_crtc(crtc, plane_id) {
6135
		struct skl_plane_wm *wm = &out->planes[plane_id];
6136

6137
		for (level = 0; level <= max_level; level++) {
6138 6139
			if (plane_id != PLANE_CURSOR)
				val = I915_READ(PLANE_WM(pipe, plane_id, level));
6140 6141
			else
				val = I915_READ(CUR_WM(pipe, level));
6142

6143
			skl_wm_level_from_reg_val(val, &wm->wm[level]);
6144 6145
		}

6146 6147 6148
		if (INTEL_GEN(dev_priv) >= 12)
			wm->sagv_wm0 = wm->wm[0];

6149 6150
		if (plane_id != PLANE_CURSOR)
			val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
6151 6152 6153 6154
		else
			val = I915_READ(CUR_WM_TRANS(pipe));

		skl_wm_level_from_reg_val(val, &wm->trans_wm);
6155 6156
	}

6157
	if (!crtc->active)
6158
		return;
6159 6160
}

6161
void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6162
{
6163
	struct intel_crtc *crtc;
6164
	struct intel_crtc_state *crtc_state;
6165

6166
	skl_ddb_get_hw_state(dev_priv);
6167
	for_each_intel_crtc(&dev_priv->drm, crtc) {
6168
		crtc_state = to_intel_crtc_state(crtc->base.state);
6169

6170
		skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6171
	}
6172

6173
	if (dev_priv->active_pipes) {
6174 6175 6176
		/* Fully recompute DDB on first atomic commit */
		dev_priv->wm.distrust_bios_wm = true;
	}
6177 6178
}

6179
static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6180
{
6181
	struct drm_device *dev = crtc->base.dev;
6182
	struct drm_i915_private *dev_priv = to_i915(dev);
6183
	struct ilk_wm_values *hw = &dev_priv->wm.hw;
6184 6185
	struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
	struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6186
	enum pipe pipe = crtc->pipe;
6187
	static const i915_reg_t wm0_pipe_reg[] = {
6188 6189 6190 6191 6192 6193 6194
		[PIPE_A] = WM0_PIPEA_ILK,
		[PIPE_B] = WM0_PIPEB_ILK,
		[PIPE_C] = WM0_PIPEC_IVB,
	};

	hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);

6195 6196
	memset(active, 0, sizeof(*active));

6197
	active->pipe_enabled = crtc->active;
6198 6199

	if (active->pipe_enabled) {
6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212
		u32 tmp = hw->wm_pipe[pipe];

		/*
		 * For active pipes LP0 watermark is marked as
		 * enabled, and LP1+ watermaks as disabled since
		 * we can't really reverse compute them in case
		 * multiple pipes are active.
		 */
		active->wm[0].enable = true;
		active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
		active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
		active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
	} else {
6213
		int level, max_level = ilk_wm_max_level(dev_priv);
6214 6215 6216 6217 6218 6219 6220 6221 6222

		/*
		 * For inactive pipes, all watermark levels
		 * should be marked as enabled but zeroed,
		 * which is what we'd compute them to.
		 */
		for (level = 0; level <= max_level; level++)
			active->wm[level].enable = true;
	}
6223

6224
	crtc->wm.active.ilk = *active;
6225 6226
}

6227 6228 6229 6230 6231
#define _FW_WM(value, plane) \
	(((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
#define _FW_WM_VLV(value, plane) \
	(((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)

6232 6233 6234
static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
			       struct g4x_wm_values *wm)
{
6235
	u32 tmp;
6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255 6256 6257

	tmp = I915_READ(DSPFW1);
	wm->sr.plane = _FW_WM(tmp, SR);
	wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);

	tmp = I915_READ(DSPFW2);
	wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
	wm->sr.fbc = _FW_WM(tmp, FBC_SR);
	wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
	wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
	wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);

	tmp = I915_READ(DSPFW3);
	wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
	wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
	wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
}

6258 6259 6260 6261
static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
			       struct vlv_wm_values *wm)
{
	enum pipe pipe;
6262
	u32 tmp;
6263 6264 6265 6266

	for_each_pipe(dev_priv, pipe) {
		tmp = I915_READ(VLV_DDL(pipe));

6267
		wm->ddl[pipe].plane[PLANE_PRIMARY] =
6268
			(tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6269
		wm->ddl[pipe].plane[PLANE_CURSOR] =
6270
			(tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6271
		wm->ddl[pipe].plane[PLANE_SPRITE0] =
6272
			(tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6273
		wm->ddl[pipe].plane[PLANE_SPRITE1] =
6274 6275 6276 6277 6278
			(tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
	}

	tmp = I915_READ(DSPFW1);
	wm->sr.plane = _FW_WM(tmp, SR);
6279 6280 6281
	wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6282 6283

	tmp = I915_READ(DSPFW2);
6284 6285 6286
	wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
	wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6287 6288 6289 6290 6291 6292

	tmp = I915_READ(DSPFW3);
	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);

	if (IS_CHERRYVIEW(dev_priv)) {
		tmp = I915_READ(DSPFW7_CHV);
6293 6294
		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6295 6296

		tmp = I915_READ(DSPFW8_CHV);
6297 6298
		wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
		wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6299 6300

		tmp = I915_READ(DSPFW9_CHV);
6301 6302
		wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
		wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6303 6304 6305

		tmp = I915_READ(DSPHOWM);
		wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6306 6307 6308 6309 6310 6311 6312 6313 6314
		wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
		wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
		wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
		wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
		wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
		wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
		wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6315 6316
	} else {
		tmp = I915_READ(DSPFW7);
6317 6318
		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6319 6320 6321

		tmp = I915_READ(DSPHOWM);
		wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6322 6323 6324 6325 6326 6327
		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
		wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
		wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
		wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
		wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6328 6329 6330 6331 6332 6333
	}
}

#undef _FW_WM
#undef _FW_WM_VLV

6334
void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6335 6336 6337 6338 6339 6340 6341 6342
{
	struct g4x_wm_values *wm = &dev_priv->wm.g4x;
	struct intel_crtc *crtc;

	g4x_read_wm_values(dev_priv, wm);

	wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;

6343
	for_each_intel_crtc(&dev_priv->drm, crtc) {
6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364 6365 6366 6367 6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400 6401 6402
		struct intel_crtc_state *crtc_state =
			to_intel_crtc_state(crtc->base.state);
		struct g4x_wm_state *active = &crtc->wm.active.g4x;
		struct g4x_pipe_wm *raw;
		enum pipe pipe = crtc->pipe;
		enum plane_id plane_id;
		int level, max_level;

		active->cxsr = wm->cxsr;
		active->hpll_en = wm->hpll_en;
		active->fbc_en = wm->fbc_en;

		active->sr = wm->sr;
		active->hpll = wm->hpll;

		for_each_plane_id_on_crtc(crtc, plane_id) {
			active->wm.plane[plane_id] =
				wm->pipe[pipe].plane[plane_id];
		}

		if (wm->cxsr && wm->hpll_en)
			max_level = G4X_WM_LEVEL_HPLL;
		else if (wm->cxsr)
			max_level = G4X_WM_LEVEL_SR;
		else
			max_level = G4X_WM_LEVEL_NORMAL;

		level = G4X_WM_LEVEL_NORMAL;
		raw = &crtc_state->wm.g4x.raw[level];
		for_each_plane_id_on_crtc(crtc, plane_id)
			raw->plane[plane_id] = active->wm.plane[plane_id];

		if (++level > max_level)
			goto out;

		raw = &crtc_state->wm.g4x.raw[level];
		raw->plane[PLANE_PRIMARY] = active->sr.plane;
		raw->plane[PLANE_CURSOR] = active->sr.cursor;
		raw->plane[PLANE_SPRITE0] = 0;
		raw->fbc = active->sr.fbc;

		if (++level > max_level)
			goto out;

		raw = &crtc_state->wm.g4x.raw[level];
		raw->plane[PLANE_PRIMARY] = active->hpll.plane;
		raw->plane[PLANE_CURSOR] = active->hpll.cursor;
		raw->plane[PLANE_SPRITE0] = 0;
		raw->fbc = active->hpll.fbc;

	out:
		for_each_plane_id_on_crtc(crtc, plane_id)
			g4x_raw_plane_wm_set(crtc_state, level,
					     plane_id, USHRT_MAX);
		g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);

		crtc_state->wm.g4x.optimal = *active;
		crtc_state->wm.g4x.intermediate = *active;

6403 6404 6405 6406 6407 6408
		drm_dbg_kms(&dev_priv->drm,
			    "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
			    pipe_name(pipe),
			    wm->pipe[pipe].plane[PLANE_PRIMARY],
			    wm->pipe[pipe].plane[PLANE_CURSOR],
			    wm->pipe[pipe].plane[PLANE_SPRITE0]);
6409 6410
	}

6411 6412 6413 6414 6415 6416 6417 6418
	drm_dbg_kms(&dev_priv->drm,
		    "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
		    wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
	drm_dbg_kms(&dev_priv->drm,
		    "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
		    wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
	drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
		    yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438
}

void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
{
	struct intel_plane *plane;
	struct intel_crtc *crtc;

	mutex_lock(&dev_priv->wm.wm_mutex);

	for_each_intel_plane(&dev_priv->drm, plane) {
		struct intel_crtc *crtc =
			intel_get_crtc_for_pipe(dev_priv, plane->pipe);
		struct intel_crtc_state *crtc_state =
			to_intel_crtc_state(crtc->base.state);
		struct intel_plane_state *plane_state =
			to_intel_plane_state(plane->base.state);
		struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
		enum plane_id plane_id = plane->id;
		int level;

6439
		if (plane_state->uapi.visible)
6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476
			continue;

		for (level = 0; level < 3; level++) {
			struct g4x_pipe_wm *raw =
				&crtc_state->wm.g4x.raw[level];

			raw->plane[plane_id] = 0;
			wm_state->wm.plane[plane_id] = 0;
		}

		if (plane_id == PLANE_PRIMARY) {
			for (level = 0; level < 3; level++) {
				struct g4x_pipe_wm *raw =
					&crtc_state->wm.g4x.raw[level];
				raw->fbc = 0;
			}

			wm_state->sr.fbc = 0;
			wm_state->hpll.fbc = 0;
			wm_state->fbc_en = false;
		}
	}

	for_each_intel_crtc(&dev_priv->drm, crtc) {
		struct intel_crtc_state *crtc_state =
			to_intel_crtc_state(crtc->base.state);

		crtc_state->wm.g4x.intermediate =
			crtc_state->wm.g4x.optimal;
		crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
	}

	g4x_program_watermarks(dev_priv);

	mutex_unlock(&dev_priv->wm.wm_mutex);
}

6477
void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6478 6479
{
	struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6480
	struct intel_crtc *crtc;
6481 6482 6483 6484 6485 6486 6487 6488
	u32 val;

	vlv_read_wm_values(dev_priv, wm);

	wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
	wm->level = VLV_WM_LEVEL_PM2;

	if (IS_CHERRYVIEW(dev_priv)) {
6489
		vlv_punit_get(dev_priv);
6490

6491
		val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6492 6493 6494
		if (val & DSP_MAXFIFO_PM5_ENABLE)
			wm->level = VLV_WM_LEVEL_PM5;

6495 6496 6497 6498 6499 6500 6501 6502 6503
		/*
		 * If DDR DVFS is disabled in the BIOS, Punit
		 * will never ack the request. So if that happens
		 * assume we don't have to enable/disable DDR DVFS
		 * dynamically. To test that just set the REQ_ACK
		 * bit to poke the Punit, but don't change the
		 * HIGH/LOW bits so that we don't actually change
		 * the current state.
		 */
6504
		val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6505 6506 6507 6508 6509
		val |= FORCE_DDR_FREQ_REQ_ACK;
		vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);

		if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
			      FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6510 6511 6512
			drm_dbg_kms(&dev_priv->drm,
				    "Punit not acking DDR DVFS request, "
				    "assuming DDR DVFS is disabled\n");
6513 6514 6515 6516 6517 6518
			dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
		} else {
			val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
			if ((val & FORCE_DDR_HIGH_FREQ) == 0)
				wm->level = VLV_WM_LEVEL_DDR_DVFS;
		}
6519

6520
		vlv_punit_put(dev_priv);
6521 6522
	}

6523
	for_each_intel_crtc(&dev_priv->drm, crtc) {
6524 6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538
		struct intel_crtc_state *crtc_state =
			to_intel_crtc_state(crtc->base.state);
		struct vlv_wm_state *active = &crtc->wm.active.vlv;
		const struct vlv_fifo_state *fifo_state =
			&crtc_state->wm.vlv.fifo_state;
		enum pipe pipe = crtc->pipe;
		enum plane_id plane_id;
		int level;

		vlv_get_fifo_size(crtc_state);

		active->num_levels = wm->level + 1;
		active->cxsr = wm->cxsr;

		for (level = 0; level < active->num_levels; level++) {
6539
			struct g4x_pipe_wm *raw =
6540 6541 6542 6543 6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554 6555 6556 6557 6558 6559 6560
				&crtc_state->wm.vlv.raw[level];

			active->sr[level].plane = wm->sr.plane;
			active->sr[level].cursor = wm->sr.cursor;

			for_each_plane_id_on_crtc(crtc, plane_id) {
				active->wm[level].plane[plane_id] =
					wm->pipe[pipe].plane[plane_id];

				raw->plane[plane_id] =
					vlv_invert_wm_value(active->wm[level].plane[plane_id],
							    fifo_state->plane[plane_id]);
			}
		}

		for_each_plane_id_on_crtc(crtc, plane_id)
			vlv_raw_plane_wm_set(crtc_state, level,
					     plane_id, USHRT_MAX);
		vlv_invalidate_wms(crtc, active, level);

		crtc_state->wm.vlv.optimal = *active;
6561
		crtc_state->wm.vlv.intermediate = *active;
6562

6563 6564 6565 6566 6567 6568 6569
		drm_dbg_kms(&dev_priv->drm,
			    "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
			    pipe_name(pipe),
			    wm->pipe[pipe].plane[PLANE_PRIMARY],
			    wm->pipe[pipe].plane[PLANE_CURSOR],
			    wm->pipe[pipe].plane[PLANE_SPRITE0],
			    wm->pipe[pipe].plane[PLANE_SPRITE1]);
6570
	}
6571

6572 6573 6574
	drm_dbg_kms(&dev_priv->drm,
		    "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
		    wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
6575 6576
}

6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593 6594 6595 6596
void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
{
	struct intel_plane *plane;
	struct intel_crtc *crtc;

	mutex_lock(&dev_priv->wm.wm_mutex);

	for_each_intel_plane(&dev_priv->drm, plane) {
		struct intel_crtc *crtc =
			intel_get_crtc_for_pipe(dev_priv, plane->pipe);
		struct intel_crtc_state *crtc_state =
			to_intel_crtc_state(crtc->base.state);
		struct intel_plane_state *plane_state =
			to_intel_plane_state(plane->base.state);
		struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
		const struct vlv_fifo_state *fifo_state =
			&crtc_state->wm.vlv.fifo_state;
		enum plane_id plane_id = plane->id;
		int level;

6597
		if (plane_state->uapi.visible)
6598 6599 6600
			continue;

		for (level = 0; level < wm_state->num_levels; level++) {
6601
			struct g4x_pipe_wm *raw =
6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622 6623 6624 6625
				&crtc_state->wm.vlv.raw[level];

			raw->plane[plane_id] = 0;

			wm_state->wm[level].plane[plane_id] =
				vlv_invert_wm_value(raw->plane[plane_id],
						    fifo_state->plane[plane_id]);
		}
	}

	for_each_intel_crtc(&dev_priv->drm, crtc) {
		struct intel_crtc_state *crtc_state =
			to_intel_crtc_state(crtc->base.state);

		crtc_state->wm.vlv.intermediate =
			crtc_state->wm.vlv.optimal;
		crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
	}

	vlv_program_watermarks(dev_priv);

	mutex_unlock(&dev_priv->wm.wm_mutex);
}

6626 6627 6628 6629 6630 6631 6632 6633 6634 6635 6636 6637 6638 6639 6640 6641
/*
 * FIXME should probably kill this and improve
 * the real watermark readout/sanitation instead
 */
static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
{
	I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
	I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
	I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);

	/*
	 * Don't touch WM1S_LP_EN here.
	 * Doing so could cause underruns.
	 */
}

6642
void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
6643
{
6644
	struct ilk_wm_values *hw = &dev_priv->wm.hw;
6645
	struct intel_crtc *crtc;
6646

6647 6648
	ilk_init_lp_watermarks(dev_priv);

6649
	for_each_intel_crtc(&dev_priv->drm, crtc)
6650 6651 6652 6653 6654 6655 6656
		ilk_pipe_wm_get_hw_state(crtc);

	hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
	hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
	hw->wm_lp[2] = I915_READ(WM3_LP_ILK);

	hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
6657
	if (INTEL_GEN(dev_priv) >= 7) {
6658 6659 6660
		hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
		hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
	}
6661

6662
	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6663 6664
		hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
			INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6665
	else if (IS_IVYBRIDGE(dev_priv))
6666 6667
		hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
			INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6668 6669 6670 6671 6672

	hw->enable_fbc_wm =
		!(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
}

6673 6674
/**
 * intel_update_watermarks - update FIFO watermark values based on current modes
6675
 * @crtc: the #intel_crtc on which to compute the WM
6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705
 *
 * Calculate watermark values for the various WM regs based on current mode
 * and plane configuration.
 *
 * There are several cases to deal with here:
 *   - normal (i.e. non-self-refresh)
 *   - self-refresh (SR) mode
 *   - lines are large relative to FIFO size (buffer can hold up to 2)
 *   - lines are small relative to FIFO size (buffer can hold more than 2
 *     lines), so need to account for TLB latency
 *
 *   The normal calculation is:
 *     watermark = dotclock * bytes per pixel * latency
 *   where latency is platform & configuration dependent (we assume pessimal
 *   values here).
 *
 *   The SR calculation is:
 *     watermark = (trunc(latency/line time)+1) * surface width *
 *       bytes per pixel
 *   where
 *     line time = htotal / dotclock
 *     surface width = hdisplay for normal plane and 64 for cursor
 *   and latency is assumed to be high, as above.
 *
 * The final value programmed to the register should always be rounded up,
 * and include an extra 2 entries to account for clock crossings.
 *
 * We don't use the sprite, so we can ignore that.  And on Crestline we have
 * to set the non-SR watermarks to 8.
 */
6706
void intel_update_watermarks(struct intel_crtc *crtc)
6707
{
6708
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6709 6710

	if (dev_priv->display.update_wm)
6711
		dev_priv->display.update_wm(crtc);
6712 6713
}

6714 6715 6716 6717
void intel_enable_ipc(struct drm_i915_private *dev_priv)
{
	u32 val;

6718 6719 6720
	if (!HAS_IPC(dev_priv))
		return;

6721 6722 6723 6724 6725 6726 6727 6728 6729 6730
	val = I915_READ(DISP_ARB_CTL2);

	if (dev_priv->ipc_enabled)
		val |= DISP_IPC_ENABLE;
	else
		val &= ~DISP_IPC_ENABLE;

	I915_WRITE(DISP_ARB_CTL2, val);
}

6731 6732 6733 6734 6735 6736 6737 6738 6739 6740 6741 6742 6743
static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
{
	/* Display WA #0477 WaDisableIPC: skl */
	if (IS_SKYLAKE(dev_priv))
		return false;

	/* Display WA #1141: SKL:all KBL:all CFL */
	if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv))
		return dev_priv->dram_info.symmetric_memory;

	return true;
}

6744 6745 6746 6747 6748
void intel_init_ipc(struct drm_i915_private *dev_priv)
{
	if (!HAS_IPC(dev_priv))
		return;

6749
	dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
6750

6751 6752 6753
	intel_enable_ipc(dev_priv);
}

6754 6755 6756 6757 6758 6759 6760 6761 6762
static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
{
	/*
	 * On Ibex Peak and Cougar Point, we need to disable clock
	 * gating for the panel power sequencer or it will fail to
	 * start up when no ports are active.
	 */
	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
}
6763

6764
static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
6765
{
6766
	enum pipe pipe;
6767

6768 6769 6770 6771
	for_each_pipe(dev_priv, pipe) {
		I915_WRITE(DSPCNTR(pipe),
			   I915_READ(DSPCNTR(pipe)) |
			   DISPPLANE_TRICKLE_FEED_DISABLE);
6772

6773 6774
		I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
		POSTING_READ(DSPSURF(pipe));
6775 6776 6777
	}
}

6778
static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
6779
{
6780
	u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6781

6782 6783 6784 6785 6786 6787 6788
	/*
	 * Required for FBC
	 * WaFbcDisableDpfcClockGating:ilk
	 */
	dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
		   ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
		   ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6789

6790 6791 6792 6793 6794
	I915_WRITE(PCH_3DCGDIS0,
		   MARIUNIT_CLOCK_GATE_DISABLE |
		   SVSMUNIT_CLOCK_GATE_DISABLE);
	I915_WRITE(PCH_3DCGDIS1,
		   VFMUNIT_CLOCK_GATE_DISABLE);
6795

6796 6797 6798 6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809
	/*
	 * According to the spec the following bits should be set in
	 * order to enable memory self-refresh
	 * The bit 22/21 of 0x42004
	 * The bit 5 of 0x42020
	 * The bit 15 of 0x45000
	 */
	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		   (I915_READ(ILK_DISPLAY_CHICKEN2) |
		    ILK_DPARB_GATE | ILK_VSDPFD_FULL));
	dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
	I915_WRITE(DISP_ARB_CTL,
		   (I915_READ(DISP_ARB_CTL) |
		    DISP_FBC_WM_DIS));
6810 6811

	/*
6812 6813 6814 6815 6816
	 * Based on the document from hardware guys the following bits
	 * should be set unconditionally in order to enable FBC.
	 * The bit 22 of 0x42000
	 * The bit 22 of 0x42004
	 * The bit 7,8,9 of 0x42020.
6817
	 */
6818 6819 6820 6821 6822 6823 6824 6825 6826
	if (IS_IRONLAKE_M(dev_priv)) {
		/* WaFbcAsynchFlipDisableFbcQueue:ilk */
		I915_WRITE(ILK_DISPLAY_CHICKEN1,
			   I915_READ(ILK_DISPLAY_CHICKEN1) |
			   ILK_FBCQ_DIS);
		I915_WRITE(ILK_DISPLAY_CHICKEN2,
			   I915_READ(ILK_DISPLAY_CHICKEN2) |
			   ILK_DPARB_GATE);
	}
6827

6828
	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6829

6830 6831 6832 6833 6834 6835
	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		   I915_READ(ILK_DISPLAY_CHICKEN2) |
		   ILK_ELPIN_409_SELECT);
	I915_WRITE(_3D_CHICKEN2,
		   _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
		   _3D_CHICKEN2_WM_READ_PIPELINED);
6836

6837 6838 6839
	/* WaDisableRenderCachePipelinedFlush:ilk */
	I915_WRITE(CACHE_MODE_0,
		   _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
6840

6841 6842
	/* WaDisable_RenderCache_OperationalFlush:ilk */
	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6843

6844
	g4x_disable_trickle_feed(dev_priv);
6845

6846
	ibx_init_clock_gating(dev_priv);
6847 6848
}

6849
static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
6850
{
6851 6852
	enum pipe pipe;
	u32 val;
6853

6854 6855 6856 6857 6858 6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881
	/*
	 * On Ibex Peak and Cougar Point, we need to disable clock
	 * gating for the panel power sequencer or it will fail to
	 * start up when no ports are active.
	 */
	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
		   PCH_DPLUNIT_CLOCK_GATE_DISABLE |
		   PCH_CPUNIT_CLOCK_GATE_DISABLE);
	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
		   DPLS_EDP_PPS_FIX_DIS);
	/* The below fixes the weird display corruption, a few pixels shifted
	 * downward, on (only) LVDS of some HP laptops with IVY.
	 */
	for_each_pipe(dev_priv, pipe) {
		val = I915_READ(TRANS_CHICKEN2(pipe));
		val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
		val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
		if (dev_priv->vbt.fdi_rx_polarity_inverted)
			val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
		val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
		val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
		I915_WRITE(TRANS_CHICKEN2(pipe), val);
	}
	/* WADP0ClockGatingDisable */
	for_each_pipe(dev_priv, pipe) {
		I915_WRITE(TRANS_CHICKEN1(pipe),
			   TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
	}
6882 6883
}

6884
static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
6885
{
6886
	u32 tmp;
6887

6888 6889
	tmp = I915_READ(MCH_SSKPD);
	if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
6890 6891 6892
		drm_dbg_kms(&dev_priv->drm,
			    "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
			    tmp);
6893 6894
}

6895
static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
6896
{
6897
	u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6898

6899
	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6900

6901 6902 6903
	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		   I915_READ(ILK_DISPLAY_CHICKEN2) |
		   ILK_ELPIN_409_SELECT);
6904

6905 6906 6907
	/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
	I915_WRITE(_3D_CHICKEN,
		   _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
6908

6909 6910
	/* WaDisable_RenderCache_OperationalFlush:snb */
	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6911

6912 6913 6914 6915 6916 6917 6918 6919 6920 6921
	/*
	 * BSpec recoomends 8x4 when MSAA is used,
	 * however in practice 16x4 seems fastest.
	 *
	 * Note that PS/WM thread counts depend on the WIZ hashing
	 * disable bit, which we don't touch here, but it's good
	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
	 */
	I915_WRITE(GEN6_GT_MODE,
		   _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6922

6923 6924
	I915_WRITE(CACHE_MODE_0,
		   _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
6925

6926 6927 6928 6929
	I915_WRITE(GEN6_UCGCTL1,
		   I915_READ(GEN6_UCGCTL1) |
		   GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
		   GEN6_CSUNIT_CLOCK_GATE_DISABLE);
6930

6931 6932 6933 6934 6935 6936 6937 6938 6939 6940 6941 6942
	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
	 * gating disable must be set.  Failure to set it results in
	 * flickering pixels due to Z write ordering failures after
	 * some amount of runtime in the Mesa "fire" demo, and Unigine
	 * Sanctuary and Tropics, and apparently anything else with
	 * alpha test or pixel discard.
	 *
	 * According to the spec, bit 11 (RCCUNIT) must also be set,
	 * but we didn't debug actual testcases to find it out.
	 *
	 * WaDisableRCCUnitClockGating:snb
	 * WaDisableRCPBUnitClockGating:snb
6943
	 */
6944 6945 6946
	I915_WRITE(GEN6_UCGCTL2,
		   GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
		   GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
6947

6948 6949 6950
	/* WaStripsFansDisableFastClipPerformanceFix:snb */
	I915_WRITE(_3D_CHICKEN3,
		   _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
6951

6952 6953 6954 6955 6956 6957 6958
	/*
	 * Bspec says:
	 * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
	 * 3DSTATE_SF number of SF output attributes is more than 16."
	 */
	I915_WRITE(_3D_CHICKEN3,
		   _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
6959

6960 6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979 6980
	/*
	 * According to the spec the following bits should be
	 * set in order to enable memory self-refresh and fbc:
	 * The bit21 and bit22 of 0x42000
	 * The bit21 and bit22 of 0x42004
	 * The bit5 and bit7 of 0x42020
	 * The bit14 of 0x70180
	 * The bit14 of 0x71180
	 *
	 * WaFbcAsynchFlipDisableFbcQueue:snb
	 */
	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		   I915_READ(ILK_DISPLAY_CHICKEN1) |
		   ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		   I915_READ(ILK_DISPLAY_CHICKEN2) |
		   ILK_DPARB_GATE | ILK_VSDPFD_FULL);
	I915_WRITE(ILK_DSPCLK_GATE_D,
		   I915_READ(ILK_DSPCLK_GATE_D) |
		   ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
		   ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
6981

6982
	g4x_disable_trickle_feed(dev_priv);
6983

6984
	cpt_init_clock_gating(dev_priv);
6985

6986
	gen6_check_mch_setup(dev_priv);
6987 6988
}

6989
static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
6990
{
6991
	u32 reg = I915_READ(GEN7_FF_THREAD_MODE);
6992

6993 6994 6995 6996 6997 6998 6999 7000 7001 7002
	/*
	 * WaVSThreadDispatchOverride:ivb,vlv
	 *
	 * This actually overrides the dispatch
	 * mode for all thread types.
	 */
	reg &= ~GEN7_FF_SCHED_MASK;
	reg |= GEN7_FF_TS_SCHED_HW;
	reg |= GEN7_FF_VS_SCHED_HW;
	reg |= GEN7_FF_DS_SCHED_HW;
7003

7004
	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
7005 7006
}

7007
static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7008
{
7009 7010 7011
	/*
	 * TODO: this bit should only be enabled when really needed, then
	 * disabled when not needed anymore in order to save power.
7012
	 */
7013
	if (HAS_PCH_LPT_LP(dev_priv))
7014 7015 7016
		I915_WRITE(SOUTH_DSPCLK_GATE_D,
			   I915_READ(SOUTH_DSPCLK_GATE_D) |
			   PCH_LP_PARTITION_LEVEL_DISABLE);
7017 7018

	/* WADPOClockGatingDisable:hsw */
7019 7020
	I915_WRITE(TRANS_CHICKEN1(PIPE_A),
		   I915_READ(TRANS_CHICKEN1(PIPE_A)) |
7021
		   TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7022 7023
}

7024
static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7025
{
7026
	if (HAS_PCH_LPT_LP(dev_priv)) {
7027
		u32 val = I915_READ(SOUTH_DSPCLK_GATE_D);
7028 7029 7030 7031 7032 7033

		val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
		I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
	}
}

7034 7035 7036 7037 7038
static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
				   int general_prio_credits,
				   int high_prio_credits)
{
	u32 misccpctl;
7039
	u32 val;
7040 7041 7042 7043 7044

	/* WaTempDisableDOPClkGating:bdw */
	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);

7045 7046 7047 7048 7049
	val = I915_READ(GEN8_L3SQCREG1);
	val &= ~L3_PRIO_CREDITS_MASK;
	val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
	val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
	I915_WRITE(GEN8_L3SQCREG1, val);
7050 7051 7052 7053 7054 7055 7056 7057 7058 7059

	/*
	 * Wait at least 100 clocks before re-enabling clock gating.
	 * See the definition of L3SQCREG1 in BSpec.
	 */
	POSTING_READ(GEN8_L3SQCREG1);
	udelay(1);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
}

7060 7061 7062 7063 7064
static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
{
	/* This is not an Wa. Enable to reduce Sampler power */
	I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
		   I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
7065

7066 7067 7068
	/*Wa_14010594013:icl, ehl */
	intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
			 0, CNL_DELAY_PMRSP);
7069 7070
}

7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084
static void tgl_init_clock_gating(struct drm_i915_private *dev_priv)
{
	u32 vd_pg_enable = 0;
	unsigned int i;

	/* This is not a WA. Enable VD HCP & MFX_ENC powergate */
	for (i = 0; i < I915_MAX_VCS; i++) {
		if (HAS_ENGINE(dev_priv, _VCS(i)))
			vd_pg_enable |= VDN_HCP_POWERGATE_ENABLE(i) |
					VDN_MFX_POWERGATE_ENABLE(i);
	}

	I915_WRITE(POWERGATE_ENABLE,
		   I915_READ(POWERGATE_ENABLE) | vd_pg_enable);
7085 7086 7087 7088 7089

	/* Wa_1409825376:tgl (pre-prod)*/
	if (IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_A0))
		I915_WRITE(GEN9_CLKGATE_DIS_3, I915_READ(GEN9_CLKGATE_DIS_3) |
			   TGL_VRH_GATING_DIS);
Matt Atwood's avatar
Matt Atwood committed
7090 7091 7092 7093

	/* Wa_14011059788:tgl */
	intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
			 0, DFR_DISABLE);
7094 7095
}

7096 7097 7098 7099 7100
static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
{
	if (!HAS_PCH_CNP(dev_priv))
		return;

7101
	/* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7102 7103
	I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
		   CNP_PWM_CGE_GATING_DISABLE);
7104 7105
}

7106
static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
7107
{
7108
	u32 val;
7109 7110
	cnp_init_clock_gating(dev_priv);

7111 7112 7113 7114
	/* This is not an Wa. Enable for better image quality */
	I915_WRITE(_3D_CHICKEN3,
		   _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));

7115 7116 7117 7118 7119 7120 7121 7122
	/* WaEnableChickenDCPR:cnl */
	I915_WRITE(GEN8_CHICKEN_DCPR_1,
		   I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);

	/* WaFbcWakeMemOn:cnl */
	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
		   DISP_FBC_MEMORY_WAKE);

7123 7124 7125 7126
	val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
	/* ReadHitWriteOnlyDisable:cnl */
	val |= RCCUNIT_CLKGATE_DIS;
	I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
7127

7128 7129 7130 7131 7132
	/* Wa_2201832410:cnl */
	val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
	val |= GWUNIT_CLKGATE_DIS;
	I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);

7133
	/* WaDisableVFclkgate:cnl */
7134
	/* WaVFUnitClockGatingDisable:cnl */
7135 7136 7137
	val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
	val |= VFUNIT_CLKGATE_DIS;
	I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
7138 7139
}

7140 7141 7142 7143 7144 7145 7146 7147 7148 7149
static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
{
	cnp_init_clock_gating(dev_priv);
	gen9_init_clock_gating(dev_priv);

	/* WaFbcNukeOnHostModify:cfl */
	I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
		   ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
}

7150
static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7151
{
7152
	gen9_init_clock_gating(dev_priv);
7153 7154 7155 7156 7157

	/* WaDisableSDEUnitClockGating:kbl */
	if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
		I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
			   GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7158 7159 7160 7161 7162

	/* WaDisableGamClockGating:kbl */
	if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
		I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
			   GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7163

7164
	/* WaFbcNukeOnHostModify:kbl */
7165 7166
	I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
		   ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7167 7168
}

7169
static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7170
{
7171
	gen9_init_clock_gating(dev_priv);
7172 7173 7174 7175

	/* WAC6entrylatency:skl */
	I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
		   FBC_LLC_FULLY_OPEN);
7176 7177 7178 7179

	/* WaFbcNukeOnHostModify:skl */
	I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
		   ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7180 7181
}

7182
static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7183
{
7184
	enum pipe pipe;
7185

7186
	/* WaSwitchSolVfFArbitrationPriority:bdw */
7187
	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7188

7189
	/* WaPsrDPAMaskVBlankInSRD:bdw */
7190 7191 7192
	I915_WRITE(CHICKEN_PAR1_1,
		   I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);

7193
	/* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7194
	for_each_pipe(dev_priv, pipe) {
7195
		I915_WRITE(CHICKEN_PIPESL_1(pipe),
7196
			   I915_READ(CHICKEN_PIPESL_1(pipe)) |
7197
			   BDW_DPRS_MASK_VBLANK_SRD);
7198
	}
7199

7200 7201 7202 7203 7204
	/* WaVSRefCountFullforceMissDisable:bdw */
	/* WaDSRefCountFullforceMissDisable:bdw */
	I915_WRITE(GEN7_FF_THREAD_MODE,
		   I915_READ(GEN7_FF_THREAD_MODE) &
		   ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7205

7206 7207
	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		   _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7208 7209 7210 7211

	/* WaDisableSDEUnitClockGating:bdw */
	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
		   GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7212

7213 7214
	/* WaProgramL3SqcReg1Default:bdw */
	gen8_set_l3sqc_credits(dev_priv, 30, 2);
7215

7216 7217 7218 7219
	/* WaKVMNotificationOnConfigChange:bdw */
	I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
		   | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);

7220
	lpt_init_clock_gating(dev_priv);
7221 7222 7223 7224 7225 7226 7227 7228

	/* WaDisableDopClockGating:bdw
	 *
	 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
	 * clock gating.
	 */
	I915_WRITE(GEN6_UCGCTL1,
		   I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7229 7230
}

7231
static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7232
{
7233
	/* This is required by WaCatErrorRejectionIssue:hsw */
7234
	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7235 7236
		   I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
		   GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7237

7238
	/* WaSwitchSolVfFArbitrationPriority:hsw */
7239 7240
	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);

7241
	lpt_init_clock_gating(dev_priv);
7242 7243
}

7244
static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7245
{
7246
	u32 snpcr;
7247

7248
	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7249

7250
	/* WaDisableEarlyCull:ivb */
7251 7252 7253
	I915_WRITE(_3D_CHICKEN3,
		   _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));

7254
	/* WaDisableBackToBackFlipFix:ivb */
7255 7256 7257 7258
	I915_WRITE(IVB_CHICKEN3,
		   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
		   CHICKEN3_DGMG_DONE_FIX_DISABLE);

7259
	/* WaDisablePSDDualDispatchEnable:ivb */
7260
	if (IS_IVB_GT1(dev_priv))
7261 7262 7263
		I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
			   _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));

7264 7265 7266
	/* WaDisable_RenderCache_OperationalFlush:ivb */
	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));

7267
	/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
7268 7269 7270
	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		   GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);

7271
	/* WaApplyL3ControlAndL3ChickenMode:ivb */
7272 7273 7274
	I915_WRITE(GEN7_L3CNTLREG1,
			GEN7_WA_FOR_GEN7_L3_CONTROL);
	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
7275
		   GEN7_WA_L3_CHICKEN_MODE);
7276
	if (IS_IVB_GT1(dev_priv))
7277 7278
		I915_WRITE(GEN7_ROW_CHICKEN2,
			   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7279 7280 7281 7282
	else {
		/* must write both registers */
		I915_WRITE(GEN7_ROW_CHICKEN2,
			   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7283 7284
		I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
			   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7285
	}
7286

7287
	/* WaForceL3Serialization:ivb */
7288 7289 7290
	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		   ~L3SQ_URB_READ_CAM_MATCH_DISABLE);

7291
	/*
7292
	 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7293
	 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7294 7295
	 */
	I915_WRITE(GEN6_UCGCTL2,
7296
		   GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7297

7298
	/* This is required by WaCatErrorRejectionIssue:ivb */
7299 7300 7301 7302
	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
			I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
			GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);

7303
	g4x_disable_trickle_feed(dev_priv);
7304 7305

	gen7_setup_fixed_func_scheduler(dev_priv);
7306

7307 7308 7309 7310 7311
	if (0) { /* causes HiZ corruption on ivb:gt1 */
		/* enable HiZ Raw Stall Optimization */
		I915_WRITE(CACHE_MODE_0_GEN7,
			   _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
	}
7312

7313
	/* WaDisable4x2SubspanOptimization:ivb */
7314 7315
	I915_WRITE(CACHE_MODE_1,
		   _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
7316

7317 7318 7319
	/*
	 * BSpec recommends 8x4 when MSAA is used,
	 * however in practice 16x4 seems fastest.
7320 7321 7322 7323
	 *
	 * Note that PS/WM thread counts depend on the WIZ hashing
	 * disable bit, which we don't touch here, but it's good
	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
7324 7325
	 */
	I915_WRITE(GEN7_GT_MODE,
7326
		   _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
7327

7328 7329 7330 7331
	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
	snpcr &= ~GEN6_MBC_SNPCR_MASK;
	snpcr |= GEN6_MBC_SNPCR_MED;
	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
7332

7333
	if (!HAS_PCH_NOP(dev_priv))
7334
		cpt_init_clock_gating(dev_priv);
7335

7336
	gen6_check_mch_setup(dev_priv);
7337 7338
}

7339
static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7340
{
7341
	/* WaDisableEarlyCull:vlv */
7342 7343 7344
	I915_WRITE(_3D_CHICKEN3,
		   _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));

7345
	/* WaDisableBackToBackFlipFix:vlv */
7346 7347 7348 7349
	I915_WRITE(IVB_CHICKEN3,
		   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
		   CHICKEN3_DGMG_DONE_FIX_DISABLE);

7350
	/* WaPsdDispatchEnable:vlv */
7351
	/* WaDisablePSDDualDispatchEnable:vlv */
7352
	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
7353 7354
		   _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
				      GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
7355

7356 7357 7358
	/* WaDisable_RenderCache_OperationalFlush:vlv */
	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));

7359
	/* WaForceL3Serialization:vlv */
7360 7361 7362
	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		   ~L3SQ_URB_READ_CAM_MATCH_DISABLE);

7363
	/* WaDisableDopClockGating:vlv */
7364 7365 7366
	I915_WRITE(GEN7_ROW_CHICKEN2,
		   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));

7367
	/* This is required by WaCatErrorRejectionIssue:vlv */
7368 7369 7370 7371
	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		   I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
		   GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);

7372 7373
	gen7_setup_fixed_func_scheduler(dev_priv);

7374
	/*
7375
	 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7376
	 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7377 7378
	 */
	I915_WRITE(GEN6_UCGCTL2,
7379
		   GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7380

7381 7382 7383 7384 7385
	/* WaDisableL3Bank2xClockGate:vlv
	 * Disabling L3 clock gating- MMIO 940c[25] = 1
	 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
	I915_WRITE(GEN7_UCGCTL4,
		   I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7386

7387 7388 7389 7390
	/*
	 * BSpec says this must be set, even though
	 * WaDisable4x2SubspanOptimization isn't listed for VLV.
	 */
7391 7392
	I915_WRITE(CACHE_MODE_1,
		   _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
7393

7394 7395 7396 7397 7398 7399 7400 7401 7402 7403 7404
	/*
	 * BSpec recommends 8x4 when MSAA is used,
	 * however in practice 16x4 seems fastest.
	 *
	 * Note that PS/WM thread counts depend on the WIZ hashing
	 * disable bit, which we don't touch here, but it's good
	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
	 */
	I915_WRITE(GEN7_GT_MODE,
		   _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));

7405 7406 7407 7408 7409 7410
	/*
	 * WaIncreaseL3CreditsForVLVB0:vlv
	 * This is the hardware default actually.
	 */
	I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);

7411
	/*
7412
	 * WaDisableVLVClockGating_VBIIssue:vlv
7413 7414 7415
	 * Disable clock gating on th GCFG unit to prevent a delay
	 * in the reporting of vblank events.
	 */
7416
	I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7417 7418
}

7419
static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7420
{
7421 7422 7423 7424 7425
	/* WaVSRefCountFullforceMissDisable:chv */
	/* WaDSRefCountFullforceMissDisable:chv */
	I915_WRITE(GEN7_FF_THREAD_MODE,
		   I915_READ(GEN7_FF_THREAD_MODE) &
		   ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7426 7427 7428 7429

	/* WaDisableSemaphoreAndSyncFlipWait:chv */
	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		   _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7430 7431 7432 7433

	/* WaDisableCSUnitClockGating:chv */
	I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
		   GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7434 7435 7436 7437

	/* WaDisableSDEUnitClockGating:chv */
	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
		   GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7438

7439 7440 7441 7442 7443 7444
	/*
	 * WaProgramL3SqcReg1Default:chv
	 * See gfxspecs/Related Documents/Performance Guide/
	 * LSQC Setting Recommendations.
	 */
	gen8_set_l3sqc_credits(dev_priv, 38, 2);
7445 7446
}

7447
static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7448
{
7449
	u32 dspclk_gate;
7450 7451 7452 7453 7454 7455 7456 7457 7458

	I915_WRITE(RENCLK_GATE_D1, 0);
	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
		   GS_UNIT_CLOCK_GATE_DISABLE |
		   CL_UNIT_CLOCK_GATE_DISABLE);
	I915_WRITE(RAMCLK_GATE_D, 0);
	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
		OVRUNIT_CLOCK_GATE_DISABLE |
		OVCUNIT_CLOCK_GATE_DISABLE;
7459
	if (IS_GM45(dev_priv))
7460 7461
		dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
7462 7463 7464 7465

	/* WaDisableRenderCachePipelinedFlush */
	I915_WRITE(CACHE_MODE_0,
		   _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
7466

7467 7468 7469
	/* WaDisable_RenderCache_OperationalFlush:g4x */
	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));

7470
	g4x_disable_trickle_feed(dev_priv);
7471 7472
}

7473
static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7474
{
7475 7476 7477 7478 7479 7480 7481 7482 7483 7484
	struct intel_uncore *uncore = &dev_priv->uncore;

	intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
	intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
	intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
	intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
	intel_uncore_write16(uncore, DEUC, 0);
	intel_uncore_write(uncore,
			   MI_ARB_STATE,
			   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7485 7486

	/* WaDisable_RenderCache_OperationalFlush:gen4 */
7487 7488 7489
	intel_uncore_write(uncore,
			   CACHE_MODE_0,
			   _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7490 7491
}

7492
static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7493 7494 7495 7496 7497 7498 7499
{
	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
		   I965_RCC_CLOCK_GATE_DISABLE |
		   I965_RCPB_CLOCK_GATE_DISABLE |
		   I965_ISC_CLOCK_GATE_DISABLE |
		   I965_FBC_CLOCK_GATE_DISABLE);
	I915_WRITE(RENCLK_GATE_D2, 0);
7500 7501
	I915_WRITE(MI_ARB_STATE,
		   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7502 7503 7504

	/* WaDisable_RenderCache_OperationalFlush:gen4 */
	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7505 7506
}

7507
static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7508 7509 7510 7511 7512 7513
{
	u32 dstate = I915_READ(D_STATE);

	dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
		DSTATE_DOT_CLOCK_GATING;
	I915_WRITE(D_STATE, dstate);
7514

7515
	if (IS_PINEVIEW(dev_priv))
7516
		I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7517 7518 7519

	/* IIR "flip pending" means done if this bit is set */
	I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7520 7521

	/* interrupts should cause a wake up from C3 */
7522
	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7523 7524 7525

	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
	I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7526 7527 7528

	I915_WRITE(MI_ARB_STATE,
		   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7529 7530
}

7531
static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7532 7533
{
	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7534 7535 7536 7537

	/* interrupts should cause a wake up from C3 */
	I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
		   _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7538 7539 7540

	I915_WRITE(MEM_MODE,
		   _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7541 7542
}

7543
static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7544
{
7545 7546 7547
	I915_WRITE(MEM_MODE,
		   _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
		   _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7548 7549
}

7550
void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7551
{
7552
	dev_priv->display.init_clock_gating(dev_priv);
7553 7554
}

7555
void intel_suspend_hw(struct drm_i915_private *dev_priv)
7556
{
7557 7558
	if (HAS_PCH_LPT(dev_priv))
		lpt_suspend_hw(dev_priv);
7559 7560
}

7561
static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7562
{
7563 7564
	drm_dbg_kms(&dev_priv->drm,
		    "No clock gating settings or workarounds applied.\n");
7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575 7576 7577
}

/**
 * intel_init_clock_gating_hooks - setup the clock gating hooks
 * @dev_priv: device private
 *
 * Setup the hooks that configure which clocks of a given platform can be
 * gated and also apply various GT and display specific workarounds for these
 * platforms. Note that some GT specific workarounds are applied separately
 * when GPU contexts or batchbuffers start their execution.
 */
void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
{
7578
	if (IS_GEN(dev_priv, 12))
7579
		dev_priv->display.init_clock_gating = tgl_init_clock_gating;
7580
	else if (IS_GEN(dev_priv, 11))
7581
		dev_priv->display.init_clock_gating = icl_init_clock_gating;
7582
	else if (IS_CANNONLAKE(dev_priv))
7583
		dev_priv->display.init_clock_gating = cnl_init_clock_gating;
7584 7585
	else if (IS_COFFEELAKE(dev_priv))
		dev_priv->display.init_clock_gating = cfl_init_clock_gating;
7586
	else if (IS_SKYLAKE(dev_priv))
7587
		dev_priv->display.init_clock_gating = skl_init_clock_gating;
7588
	else if (IS_KABYLAKE(dev_priv))
7589
		dev_priv->display.init_clock_gating = kbl_init_clock_gating;
7590
	else if (IS_BROXTON(dev_priv))
7591
		dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7592 7593
	else if (IS_GEMINILAKE(dev_priv))
		dev_priv->display.init_clock_gating = glk_init_clock_gating;
7594
	else if (IS_BROADWELL(dev_priv))
7595
		dev_priv->display.init_clock_gating = bdw_init_clock_gating;
7596
	else if (IS_CHERRYVIEW(dev_priv))
7597
		dev_priv->display.init_clock_gating = chv_init_clock_gating;
7598
	else if (IS_HASWELL(dev_priv))
7599
		dev_priv->display.init_clock_gating = hsw_init_clock_gating;
7600
	else if (IS_IVYBRIDGE(dev_priv))
7601
		dev_priv->display.init_clock_gating = ivb_init_clock_gating;
7602
	else if (IS_VALLEYVIEW(dev_priv))
7603
		dev_priv->display.init_clock_gating = vlv_init_clock_gating;
7604
	else if (IS_GEN(dev_priv, 6))
7605
		dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7606
	else if (IS_GEN(dev_priv, 5))
7607
		dev_priv->display.init_clock_gating = ilk_init_clock_gating;
7608 7609
	else if (IS_G4X(dev_priv))
		dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7610
	else if (IS_I965GM(dev_priv))
7611
		dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
7612
	else if (IS_I965G(dev_priv))
7613
		dev_priv->display.init_clock_gating = i965g_init_clock_gating;
7614
	else if (IS_GEN(dev_priv, 3))
7615 7616 7617
		dev_priv->display.init_clock_gating = gen3_init_clock_gating;
	else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
		dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7618
	else if (IS_GEN(dev_priv, 2))
7619 7620 7621 7622 7623 7624 7625
		dev_priv->display.init_clock_gating = i830_init_clock_gating;
	else {
		MISSING_CASE(INTEL_DEVID(dev_priv));
		dev_priv->display.init_clock_gating = nop_init_clock_gating;
	}
}

7626
/* Set up chip specific power management-related functions */
7627
void intel_init_pm(struct drm_i915_private *dev_priv)
7628
{
7629
	/* For cxsr */
7630
	if (IS_PINEVIEW(dev_priv))
7631
		pnv_get_mem_freq(dev_priv);
7632
	else if (IS_GEN(dev_priv, 5))
7633
		ilk_get_mem_freq(dev_priv);
7634

7635 7636 7637
	if (intel_has_sagv(dev_priv))
		skl_setup_sagv_block_time(dev_priv);

7638
	/* For FIFO watermark updates */
7639
	if (INTEL_GEN(dev_priv) >= 9) {
7640
		skl_setup_wm_latency(dev_priv);
7641
		dev_priv->display.compute_global_watermarks = skl_compute_wm;
7642
	} else if (HAS_PCH_SPLIT(dev_priv)) {
7643
		ilk_setup_wm_latency(dev_priv);
7644

7645
		if ((IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[1] &&
7646
		     dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
7647
		    (!IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[0] &&
7648
		     dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
7649
			dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
7650 7651 7652 7653 7654 7655
			dev_priv->display.compute_intermediate_wm =
				ilk_compute_intermediate_wm;
			dev_priv->display.initial_watermarks =
				ilk_initial_watermarks;
			dev_priv->display.optimize_watermarks =
				ilk_optimize_watermarks;
7656
		} else {
7657 7658 7659
			drm_dbg_kms(&dev_priv->drm,
				    "Failed to read display plane latency. "
				    "Disable CxSR\n");
7660
		}
7661
	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
7662
		vlv_setup_wm_latency(dev_priv);
7663
		dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
7664
		dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
7665
		dev_priv->display.initial_watermarks = vlv_initial_watermarks;
7666
		dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
7667
		dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
7668 7669 7670 7671 7672 7673
	} else if (IS_G4X(dev_priv)) {
		g4x_setup_wm_latency(dev_priv);
		dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
		dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
		dev_priv->display.initial_watermarks = g4x_initial_watermarks;
		dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
7674
	} else if (IS_PINEVIEW(dev_priv)) {
7675
		if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
7676 7677 7678
					    dev_priv->is_ddr3,
					    dev_priv->fsb_freq,
					    dev_priv->mem_freq)) {
7679 7680
			drm_info(&dev_priv->drm,
				 "failed to find known CxSR latency "
7681 7682 7683 7684 7685
				 "(found ddr%s fsb freq %d, mem freq %d), "
				 "disabling CxSR\n",
				 (dev_priv->is_ddr3 == 1) ? "3" : "2",
				 dev_priv->fsb_freq, dev_priv->mem_freq);
			/* Disable CxSR and never update its watermark again */
7686
			intel_set_memory_cxsr(dev_priv, false);
7687 7688
			dev_priv->display.update_wm = NULL;
		} else
7689
			dev_priv->display.update_wm = pnv_update_wm;
7690
	} else if (IS_GEN(dev_priv, 4)) {
7691
		dev_priv->display.update_wm = i965_update_wm;
7692
	} else if (IS_GEN(dev_priv, 3)) {
7693 7694
		dev_priv->display.update_wm = i9xx_update_wm;
		dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
7695
	} else if (IS_GEN(dev_priv, 2)) {
7696
		if (INTEL_NUM_PIPES(dev_priv) == 1) {
7697
			dev_priv->display.update_wm = i845_update_wm;
7698
			dev_priv->display.get_fifo_size = i845_get_fifo_size;
7699 7700
		} else {
			dev_priv->display.update_wm = i9xx_update_wm;
7701
			dev_priv->display.get_fifo_size = i830_get_fifo_size;
7702 7703
		}
	} else {
7704 7705
		drm_err(&dev_priv->drm,
			"unexpected fall-through in %s\n", __func__);
7706 7707 7708
	}
}

7709
void intel_pm_setup(struct drm_i915_private *dev_priv)
7710
{
7711 7712
	dev_priv->runtime_pm.suspended = false;
	atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
7713
}