intel_pm.c 243 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
/*
 * Copyright © 2012 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eugeni Dodonov <eugeni.dodonov@intel.com>
 *
 */

28
#include <linux/module.h>
29
#include <linux/pm_runtime.h>
30 31 32

#include <drm/drm_atomic_helper.h>
#include <drm/drm_fourcc.h>
33
#include <drm/drm_plane_helper.h>
34

35
#include "display/intel_atomic.h"
36
#include "display/intel_atomic_plane.h"
37
#include "display/intel_bw.h"
38
#include "display/intel_de.h"
39
#include "display/intel_display_types.h"
40 41
#include "display/intel_fbc.h"
#include "display/intel_sprite.h"
42
#include "display/skl_universal_plane.h"
43

44 45
#include "gt/intel_llc.h"

46
#include "i915_drv.h"
47
#include "i915_fixed.h"
48
#include "i915_irq.h"
49
#include "i915_trace.h"
50
#include "intel_pm.h"
51
#include "intel_sideband.h"
52
#include "../../../platform/x86/intel_ips.h"
53

54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76
/* Stores plane specific WM parameters */
struct skl_wm_params {
	bool x_tiled, y_tiled;
	bool rc_surface;
	bool is_planar;
	u32 width;
	u8 cpp;
	u32 plane_pixel_rate;
	u32 y_min_scanlines;
	u32 plane_bytes_per_line;
	uint_fixed_16_16_t plane_blocks_per_line;
	uint_fixed_16_16_t y_tile_minimum;
	u32 linetime_us;
	u32 dbuf_block_size;
};

/* used in computing the new watermarks state */
struct intel_wm_config {
	unsigned int num_pipes_active;
	bool sprites_enabled;
	bool sprites_scaled;
};

77
static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
78
{
79 80 81
	if (HAS_LLC(dev_priv)) {
		/*
		 * WaCompressedResourceDisplayNewHashMode:skl,kbl
82
		 * Display WA #0390: skl,kbl
83 84 85 86
		 *
		 * Must match Sampler, Pixel Back End, and Media. See
		 * WaCompressedResourceSamplerPbeMediaNewHashMode.
		 */
87 88
		intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
			   intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) |
89 90 91
			   SKL_DE_COMPRESSED_HASH_MODE);
	}

92
	/* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
93 94
	intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
		   intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
95

96
	/* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
97 98
	intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
		   intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
99

100 101 102 103
	/*
	 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
	 * Display WA #0859: skl,bxt,kbl,glk,cfl
	 */
104
	intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
105
		   DISP_FBC_MEMORY_WAKE);
106 107
}

108
static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
109
{
110
	gen9_init_clock_gating(dev_priv);
111

112
	/* WaDisableSDEUnitClockGating:bxt */
113
	intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
114 115
		   GEN8_SDEUNIT_CLOCK_GATE_DISABLE);

116 117
	/*
	 * FIXME:
118
	 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
119
	 */
120
	intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
121
		   GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
122 123 124 125 126

	/*
	 * Wa: Backlight PWM may stop in the asserted state, causing backlight
	 * to stay fully on.
	 */
127
	intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
128
		   PWM1_GATING_DIS | PWM2_GATING_DIS);
129 130 131 132 133 134 135

	/*
	 * Lower the display internal timeout.
	 * This is needed to avoid any hard hangs when DSI port PLL
	 * is off and a MMIO access is attempted by any privilege
	 * application, using batch buffers or any other means.
	 */
136
	intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
137

138 139 140 141
	/*
	 * WaFbcTurnOffFbcWatermark:bxt
	 * Display WA #0562: bxt
	 */
142
	intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
143
		   DISP_FBC_WM_DIS);
144

145 146 147 148
	/*
	 * WaFbcHighMemBwCorruptionAvoidance:bxt
	 * Display WA #0883: bxt
	 */
149
	intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
150
		   ILK_DPFC_DISABLE_DUMMY0);
151 152
}

153 154 155 156 157 158 159 160 161
static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
{
	gen9_init_clock_gating(dev_priv);

	/*
	 * WaDisablePWMClockGating:glk
	 * Backlight PWM may stop in the asserted state, causing backlight
	 * to stay fully on.
	 */
162
	intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
163 164 165
		   PWM1_GATING_DIS | PWM2_GATING_DIS);
}

166
static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
167 168 169
{
	u32 tmp;

170
	tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199

	switch (tmp & CLKCFG_FSB_MASK) {
	case CLKCFG_FSB_533:
		dev_priv->fsb_freq = 533; /* 133*4 */
		break;
	case CLKCFG_FSB_800:
		dev_priv->fsb_freq = 800; /* 200*4 */
		break;
	case CLKCFG_FSB_667:
		dev_priv->fsb_freq =  667; /* 167*4 */
		break;
	case CLKCFG_FSB_400:
		dev_priv->fsb_freq = 400; /* 100*4 */
		break;
	}

	switch (tmp & CLKCFG_MEM_MASK) {
	case CLKCFG_MEM_533:
		dev_priv->mem_freq = 533;
		break;
	case CLKCFG_MEM_667:
		dev_priv->mem_freq = 667;
		break;
	case CLKCFG_MEM_800:
		dev_priv->mem_freq = 800;
		break;
	}

	/* detect pineview DDR3 setting */
200
	tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
201 202 203
	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
}

204
static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
205 206 207
{
	u16 ddrpll, csipll;

208 209
	ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
	csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
210 211 212 213 214 215 216 217 218 219 220 221 222 223 224

	switch (ddrpll & 0xff) {
	case 0xc:
		dev_priv->mem_freq = 800;
		break;
	case 0x10:
		dev_priv->mem_freq = 1066;
		break;
	case 0x14:
		dev_priv->mem_freq = 1333;
		break;
	case 0x18:
		dev_priv->mem_freq = 1600;
		break;
	default:
225 226
		drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
			ddrpll & 0xff);
227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
		dev_priv->mem_freq = 0;
		break;
	}

	switch (csipll & 0x3ff) {
	case 0x00c:
		dev_priv->fsb_freq = 3200;
		break;
	case 0x00e:
		dev_priv->fsb_freq = 3733;
		break;
	case 0x010:
		dev_priv->fsb_freq = 4266;
		break;
	case 0x012:
		dev_priv->fsb_freq = 4800;
		break;
	case 0x014:
		dev_priv->fsb_freq = 5333;
		break;
	case 0x016:
		dev_priv->fsb_freq = 5866;
		break;
	case 0x018:
		dev_priv->fsb_freq = 6400;
		break;
	default:
254 255
		drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
			csipll & 0x3ff);
256 257 258 259 260
		dev_priv->fsb_freq = 0;
		break;
	}
}

261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
static const struct cxsr_latency cxsr_latency_table[] = {
	{1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
	{1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
	{1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
	{1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
	{1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */

	{1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
	{1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
	{1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
	{1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
	{1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */

	{1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
	{1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
	{1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
	{1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
	{1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */

	{0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
	{0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
	{0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
	{0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
	{0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */

	{0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
	{0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
	{0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
	{0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
	{0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */

	{0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
	{0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
	{0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
	{0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
	{0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
};

299 300
static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
							 bool is_ddr3,
301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322
							 int fsb,
							 int mem)
{
	const struct cxsr_latency *latency;
	int i;

	if (fsb == 0 || mem == 0)
		return NULL;

	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		latency = &cxsr_latency_table[i];
		if (is_desktop == latency->is_desktop &&
		    is_ddr3 == latency->is_ddr3 &&
		    fsb == latency->fsb_freq && mem == latency->mem_freq)
			return latency;
	}

	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");

	return NULL;
}

323 324 325 326
static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
{
	u32 val;

327
	vlv_punit_get(dev_priv);
328 329 330 331 332 333 334 335 336 337 338 339

	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
	if (enable)
		val &= ~FORCE_DDR_HIGH_FREQ;
	else
		val |= FORCE_DDR_HIGH_FREQ;
	val &= ~FORCE_DDR_LOW_FREQ;
	val |= FORCE_DDR_FREQ_REQ_ACK;
	vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);

	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
		      FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
340 341
		drm_err(&dev_priv->drm,
			"timed out waiting for Punit DDR DVFS request\n");
342

343
	vlv_punit_put(dev_priv);
344 345
}

346 347 348 349
static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
{
	u32 val;

350
	vlv_punit_get(dev_priv);
351

352
	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
353 354 355 356
	if (enable)
		val |= DSP_MAXFIFO_PM5_ENABLE;
	else
		val &= ~DSP_MAXFIFO_PM5_ENABLE;
357
	vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
358

359
	vlv_punit_put(dev_priv);
360 361
}

362 363 364
#define FW_WM(value, plane) \
	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)

365
static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
366
{
367
	bool was_enabled;
368
	u32 val;
369

370
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
371 372 373
		was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
		intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
		intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
374
	} else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
375 376 377
		was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
		intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
		intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
378
	} else if (IS_PINEVIEW(dev_priv)) {
379
		val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
380 381 382 383 384
		was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
		if (enable)
			val |= PINEVIEW_SELF_REFRESH_EN;
		else
			val &= ~PINEVIEW_SELF_REFRESH_EN;
385 386
		intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
		intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
387
	} else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
388
		was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
389 390
		val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
			       _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
391 392
		intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
		intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
393
	} else if (IS_I915GM(dev_priv)) {
394 395 396 397 398
		/*
		 * FIXME can't find a bit like this for 915G, and
		 * and yet it does have the related watermark in
		 * FW_BLC_SELF. What's going on?
		 */
399
		was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
400 401
		val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
			       _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
402 403
		intel_uncore_write(&dev_priv->uncore, INSTPM, val);
		intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
404
	} else {
405
		return false;
406
	}
407

408 409
	trace_intel_memory_cxsr(dev_priv, was_enabled, enable);

410 411 412
	drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
		    enableddisabled(enable),
		    enableddisabled(was_enabled));
413 414

	return was_enabled;
415 416
}

Ville Syrjälä's avatar
Ville Syrjälä committed
417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453
/**
 * intel_set_memory_cxsr - Configure CxSR state
 * @dev_priv: i915 device
 * @enable: Allow vs. disallow CxSR
 *
 * Allow or disallow the system to enter a special CxSR
 * (C-state self refresh) state. What typically happens in CxSR mode
 * is that several display FIFOs may get combined into a single larger
 * FIFO for a particular plane (so called max FIFO mode) to allow the
 * system to defer memory fetches longer, and the memory will enter
 * self refresh.
 *
 * Note that enabling CxSR does not guarantee that the system enter
 * this special mode, nor does it guarantee that the system stays
 * in that mode once entered. So this just allows/disallows the system
 * to autonomously utilize the CxSR mode. Other factors such as core
 * C-states will affect when/if the system actually enters/exits the
 * CxSR mode.
 *
 * Note that on VLV/CHV this actually only controls the max FIFO mode,
 * and the system is free to enter/exit memory self refresh at any time
 * even when the use of CxSR has been disallowed.
 *
 * While the system is actually in the CxSR/max FIFO mode, some plane
 * control registers will not get latched on vblank. Thus in order to
 * guarantee the system will respond to changes in the plane registers
 * we must always disallow CxSR prior to making changes to those registers.
 * Unfortunately the system will re-evaluate the CxSR conditions at
 * frame start which happens after vblank start (which is when the plane
 * registers would get latched), so we can't proceed with the plane update
 * during the same frame where we disallowed CxSR.
 *
 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 * the hardware w.r.t. HPLL SR when writing to plane registers.
 * Disallowing just CxSR is sufficient.
 */
454
bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
455
{
456 457
	bool ret;

458
	mutex_lock(&dev_priv->wm.wm_mutex);
459
	ret = _intel_set_memory_cxsr(dev_priv, enable);
460 461 462 463
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
		dev_priv->wm.vlv.cxsr = enable;
	else if (IS_G4X(dev_priv))
		dev_priv->wm.g4x.cxsr = enable;
464
	mutex_unlock(&dev_priv->wm.wm_mutex);
465 466

	return ret;
467
}
468

469 470 471 472 473 474 475 476 477 478 479 480 481 482
/*
 * Latency for FIFO fetches is dependent on several factors:
 *   - memory configuration (speed, channels)
 *   - chipset
 *   - current MCH state
 * It can be fairly high in some situations, so here we assume a fairly
 * pessimal value.  It's a tradeoff between extra memory fetches (if we
 * set this value too high, the FIFO will fetch frequently to stay full)
 * and power consumption (set it too low to save power and we might see
 * FIFO underruns and display "flicker").
 *
 * A value of 5us seems to be a good balance; safe for very low end
 * platforms but not overly aggressive on lower latency configs.
 */
483
static const int pessimal_latency_ns = 5000;
484

485 486 487
#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
	((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))

488
static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
489
{
490
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
491
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
492
	struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
493 494
	enum pipe pipe = crtc->pipe;
	int sprite0_start, sprite1_start;
495
	u32 dsparb, dsparb2, dsparb3;
496

497
	switch (pipe) {
498
	case PIPE_A:
499 500
		dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
		dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
501 502 503 504
		sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
		sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
		break;
	case PIPE_B:
505 506
		dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
		dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
507 508 509 510
		sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
		sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
		break;
	case PIPE_C:
511 512
		dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
		dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
513 514 515 516
		sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
		sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
		break;
	default:
517 518
		MISSING_CASE(pipe);
		return;
519 520
	}

521 522 523 524
	fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
	fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
	fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
	fifo_state->plane[PLANE_CURSOR] = 63;
525 526
}

527 528
static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
			      enum i9xx_plane_id i9xx_plane)
529
{
530
	u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
531 532 533
	int size;

	size = dsparb & 0x7f;
534
	if (i9xx_plane == PLANE_B)
535 536
		size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;

537 538
	drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
		    dsparb, plane_name(i9xx_plane), size);
539 540 541 542

	return size;
}

543 544
static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
			      enum i9xx_plane_id i9xx_plane)
545
{
546
	u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
547 548 549
	int size;

	size = dsparb & 0x1ff;
550
	if (i9xx_plane == PLANE_B)
551 552 553
		size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
	size >>= 1; /* Convert to cachelines */

554 555
	drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
		    dsparb, plane_name(i9xx_plane), size);
556 557 558 559

	return size;
}

560 561
static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
			      enum i9xx_plane_id i9xx_plane)
562
{
563
	u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
564 565 566 567 568
	int size;

	size = dsparb & 0x7f;
	size >>= 2; /* Convert to cachelines */

569 570
	drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
		    dsparb, plane_name(i9xx_plane), size);
571 572 573 574 575

	return size;
}

/* Pineview has different values for various configs */
576
static const struct intel_watermark_params pnv_display_wm = {
577 578 579 580 581
	.fifo_size = PINEVIEW_DISPLAY_FIFO,
	.max_wm = PINEVIEW_MAX_WM,
	.default_wm = PINEVIEW_DFT_WM,
	.guard_size = PINEVIEW_GUARD_WM,
	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
582
};
583 584

static const struct intel_watermark_params pnv_display_hplloff_wm = {
585 586 587 588 589
	.fifo_size = PINEVIEW_DISPLAY_FIFO,
	.max_wm = PINEVIEW_MAX_WM,
	.default_wm = PINEVIEW_DFT_HPLLOFF_WM,
	.guard_size = PINEVIEW_GUARD_WM,
	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
590
};
591 592

static const struct intel_watermark_params pnv_cursor_wm = {
593 594 595 596 597
	.fifo_size = PINEVIEW_CURSOR_FIFO,
	.max_wm = PINEVIEW_CURSOR_MAX_WM,
	.default_wm = PINEVIEW_CURSOR_DFT_WM,
	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
598
};
599 600

static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
601 602 603 604 605
	.fifo_size = PINEVIEW_CURSOR_FIFO,
	.max_wm = PINEVIEW_CURSOR_MAX_WM,
	.default_wm = PINEVIEW_CURSOR_DFT_WM,
	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
606
};
607

608
static const struct intel_watermark_params i965_cursor_wm_info = {
609 610 611 612 613
	.fifo_size = I965_CURSOR_FIFO,
	.max_wm = I965_CURSOR_MAX_WM,
	.default_wm = I965_CURSOR_DFT_WM,
	.guard_size = 2,
	.cacheline_size = I915_FIFO_LINE_SIZE,
614
};
615

616
static const struct intel_watermark_params i945_wm_info = {
617 618 619 620 621
	.fifo_size = I945_FIFO_SIZE,
	.max_wm = I915_MAX_WM,
	.default_wm = 1,
	.guard_size = 2,
	.cacheline_size = I915_FIFO_LINE_SIZE,
622
};
623

624
static const struct intel_watermark_params i915_wm_info = {
625 626 627 628 629
	.fifo_size = I915_FIFO_SIZE,
	.max_wm = I915_MAX_WM,
	.default_wm = 1,
	.guard_size = 2,
	.cacheline_size = I915_FIFO_LINE_SIZE,
630
};
631

632
static const struct intel_watermark_params i830_a_wm_info = {
633 634 635 636 637
	.fifo_size = I855GM_FIFO_SIZE,
	.max_wm = I915_MAX_WM,
	.default_wm = 1,
	.guard_size = 2,
	.cacheline_size = I830_FIFO_LINE_SIZE,
638
};
639

640 641 642 643 644 645 646
static const struct intel_watermark_params i830_bc_wm_info = {
	.fifo_size = I855GM_FIFO_SIZE,
	.max_wm = I915_MAX_WM/2,
	.default_wm = 1,
	.guard_size = 2,
	.cacheline_size = I830_FIFO_LINE_SIZE,
};
647

648
static const struct intel_watermark_params i845_wm_info = {
649 650 651 652 653
	.fifo_size = I830_FIFO_SIZE,
	.max_wm = I915_MAX_WM,
	.default_wm = 1,
	.guard_size = 2,
	.cacheline_size = I830_FIFO_LINE_SIZE,
654 655
};

656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692
/**
 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 * @pixel_rate: Pipe pixel rate in kHz
 * @cpp: Plane bytes per pixel
 * @latency: Memory wakeup latency in 0.1us units
 *
 * Compute the watermark using the method 1 or "small buffer"
 * formula. The caller may additonally add extra cachelines
 * to account for TLB misses and clock crossings.
 *
 * This method is concerned with the short term drain rate
 * of the FIFO, ie. it does not account for blanking periods
 * which would effectively reduce the average drain rate across
 * a longer period. The name "small" refers to the fact the
 * FIFO is relatively small compared to the amount of data
 * fetched.
 *
 * The FIFO level vs. time graph might look something like:
 *
 *   |\   |\
 *   | \  | \
 * __---__---__ (- plane active, _ blanking)
 * -> time
 *
 * or perhaps like this:
 *
 *   |\|\  |\|\
 * __----__----__ (- plane active, _ blanking)
 * -> time
 *
 * Returns:
 * The watermark in bytes
 */
static unsigned int intel_wm_method1(unsigned int pixel_rate,
				     unsigned int cpp,
				     unsigned int latency)
{
693
	u64 ret;
694

695
	ret = mul_u32_u32(pixel_rate, cpp * latency);
696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751
	ret = DIV_ROUND_UP_ULL(ret, 10000);

	return ret;
}

/**
 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 * @pixel_rate: Pipe pixel rate in kHz
 * @htotal: Pipe horizontal total
 * @width: Plane width in pixels
 * @cpp: Plane bytes per pixel
 * @latency: Memory wakeup latency in 0.1us units
 *
 * Compute the watermark using the method 2 or "large buffer"
 * formula. The caller may additonally add extra cachelines
 * to account for TLB misses and clock crossings.
 *
 * This method is concerned with the long term drain rate
 * of the FIFO, ie. it does account for blanking periods
 * which effectively reduce the average drain rate across
 * a longer period. The name "large" refers to the fact the
 * FIFO is relatively large compared to the amount of data
 * fetched.
 *
 * The FIFO level vs. time graph might look something like:
 *
 *    |\___       |\___
 *    |    \___   |    \___
 *    |        \  |        \
 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 * -> time
 *
 * Returns:
 * The watermark in bytes
 */
static unsigned int intel_wm_method2(unsigned int pixel_rate,
				     unsigned int htotal,
				     unsigned int width,
				     unsigned int cpp,
				     unsigned int latency)
{
	unsigned int ret;

	/*
	 * FIXME remove once all users are computing
	 * watermarks in the correct place.
	 */
	if (WARN_ON_ONCE(htotal == 0))
		htotal = 1;

	ret = (latency * pixel_rate) / (htotal * 10000);
	ret = (ret + 1) * width * cpp;

	return ret;
}

752 753
/**
 * intel_calculate_wm - calculate watermark level
754
 * @pixel_rate: pixel clock
755
 * @wm: chip FIFO params
756
 * @fifo_size: size of the FIFO buffer
757
 * @cpp: bytes per pixel
758 759 760 761 762 763 764 765 766 767 768 769 770
 * @latency_ns: memory latency for the platform
 *
 * Calculate the watermark level (the level at which the display plane will
 * start fetching from memory again).  Each chip has a different display
 * FIFO size and allocation, so the caller needs to figure that out and pass
 * in the correct intel_watermark_params structure.
 *
 * As the pixel clock runs, the FIFO will be drained at a rate that depends
 * on the pixel size.  When it reaches the watermark level, it'll start
 * fetching FIFO line sized based chunks from memory until the FIFO fills
 * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 * will occur, and a display engine hang could result.
 */
771 772 773 774
static unsigned int intel_calculate_wm(int pixel_rate,
				       const struct intel_watermark_params *wm,
				       int fifo_size, int cpp,
				       unsigned int latency_ns)
775
{
776
	int entries, wm_size;
777 778 779 780 781 782 783

	/*
	 * Note: we need to make sure we don't overflow for various clock &
	 * latency values.
	 * clocks go from a few thousand to several hundred thousand.
	 * latency is usually a few thousand
	 */
784 785 786 787 788
	entries = intel_wm_method1(pixel_rate, cpp,
				   latency_ns / 100);
	entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
		wm->guard_size;
	DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
789

790 791
	wm_size = fifo_size - entries;
	DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
792 793

	/* Don't promote wm_size to unsigned... */
794
	if (wm_size > wm->max_wm)
795 796 797
		wm_size = wm->max_wm;
	if (wm_size <= 0)
		wm_size = wm->default_wm;
798 799 800 801 802 803 804 805 806 807 808

	/*
	 * Bspec seems to indicate that the value shouldn't be lower than
	 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
	 * Lets go for 8 which is the burst size since certain platforms
	 * already use a hardcoded 8 (which is what the spec says should be
	 * done).
	 */
	if (wm_size <= 8)
		wm_size = 8;

809 810 811
	return wm_size;
}

812 813 814 815 816 817 818 819 820 821
static bool is_disabling(int old, int new, int threshold)
{
	return old >= threshold && new < threshold;
}

static bool is_enabling(int old, int new, int threshold)
{
	return old < threshold && new >= threshold;
}

822 823 824 825 826
static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
{
	return dev_priv->wm.max_level + 1;
}

827 828 829
static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
				   const struct intel_plane_state *plane_state)
{
830
	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
831 832

	/* FIXME check the 'enable' instead */
833
	if (!crtc_state->hw.active)
834 835 836 837 838 839 840 841 842 843 844
		return false;

	/*
	 * Treat cursor with fb as always visible since cursor updates
	 * can happen faster than the vrefresh rate, and the current
	 * watermark code doesn't handle that correctly. Cursor updates
	 * which set/clear the fb or change the cursor size are going
	 * to get throttled by intel_legacy_cursor_update() to work
	 * around this problem with the watermark code.
	 */
	if (plane->id == PLANE_CURSOR)
845
		return plane_state->hw.fb != NULL;
846
	else
847
		return plane_state->uapi.visible;
848 849
}

850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868
static bool intel_crtc_active(struct intel_crtc *crtc)
{
	/* Be paranoid as we can arrive here with only partial
	 * state retrieved from the hardware during setup.
	 *
	 * We can ditch the adjusted_mode.crtc_clock check as soon
	 * as Haswell has gained clock readout/fastboot support.
	 *
	 * We can ditch the crtc->primary->state->fb check as soon as we can
	 * properly reconstruct framebuffers.
	 *
	 * FIXME: The intel_crtc->active here should be switched to
	 * crtc->state->active once we have proper CRTC states wired up
	 * for atomic.
	 */
	return crtc->active && crtc->base.primary->state->fb &&
		crtc->config->hw.adjusted_mode.crtc_clock;
}

869
static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
870
{
871
	struct intel_crtc *crtc, *enabled = NULL;
872

873
	for_each_intel_crtc(&dev_priv->drm, crtc) {
874
		if (intel_crtc_active(crtc)) {
875 876 877 878 879 880 881 882 883
			if (enabled)
				return NULL;
			enabled = crtc;
		}
	}

	return enabled;
}

884
static void pnv_update_wm(struct intel_crtc *unused_crtc)
885
{
886
	struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
887
	struct intel_crtc *crtc;
888 889
	const struct cxsr_latency *latency;
	u32 reg;
890
	unsigned int wm;
891

892
	latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
893 894 895
					 dev_priv->is_ddr3,
					 dev_priv->fsb_freq,
					 dev_priv->mem_freq);
896
	if (!latency) {
897 898
		drm_dbg_kms(&dev_priv->drm,
			    "Unknown FSB/MEM found, disable CxSR\n");
899
		intel_set_memory_cxsr(dev_priv, false);
900 901 902
		return;
	}

903
	crtc = single_enabled_crtc(dev_priv);
904
	if (crtc) {
905 906
		const struct drm_display_mode *pipe_mode =
			&crtc->config->hw.pipe_mode;
907 908
		const struct drm_framebuffer *fb =
			crtc->base.primary->state->fb;
909
		int cpp = fb->format->cpp[0];
910
		int clock = pipe_mode->crtc_clock;
911 912

		/* Display SR */
913 914
		wm = intel_calculate_wm(clock, &pnv_display_wm,
					pnv_display_wm.fifo_size,
915
					cpp, latency->display_sr);
916
		reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
917
		reg &= ~DSPFW_SR_MASK;
918
		reg |= FW_WM(wm, SR);
919
		intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
920
		drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
921 922

		/* cursor SR */
923 924
		wm = intel_calculate_wm(clock, &pnv_cursor_wm,
					pnv_display_wm.fifo_size,
925
					4, latency->cursor_sr);
926
		reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
927
		reg &= ~DSPFW_CURSOR_SR_MASK;
928
		reg |= FW_WM(wm, CURSOR_SR);
929
		intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
930 931

		/* Display HPLL off SR */
932 933
		wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
					pnv_display_hplloff_wm.fifo_size,
934
					cpp, latency->display_hpll_disable);
935
		reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
936
		reg &= ~DSPFW_HPLL_SR_MASK;
937
		reg |= FW_WM(wm, HPLL_SR);
938
		intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
939 940

		/* cursor HPLL off SR */
941 942
		wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
					pnv_display_hplloff_wm.fifo_size,
943
					4, latency->cursor_hpll_disable);
944
		reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
945
		reg &= ~DSPFW_HPLL_CURSOR_MASK;
946
		reg |= FW_WM(wm, HPLL_CURSOR);
947
		intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
948
		drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
949

950
		intel_set_memory_cxsr(dev_priv, true);
951
	} else {
952
		intel_set_memory_cxsr(dev_priv, false);
953 954 955
	}
}

956 957 958 959 960 961 962 963 964 965
/*
 * Documentation says:
 * "If the line size is small, the TLB fetches can get in the way of the
 *  data fetches, causing some lag in the pixel data return which is not
 *  accounted for in the above formulas. The following adjustment only
 *  needs to be applied if eight whole lines fit in the buffer at once.
 *  The WM is adjusted upwards by the difference between the FIFO size
 *  and the size of 8 whole lines. This adjustment is always performed
 *  in the actual pixel depth regardless of whether FBC is enabled or not."
 */
966
static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
967 968 969 970 971 972
{
	int tlb_miss = fifo_size * 64 - width * cpp * 8;

	return max(0, tlb_miss);
}

973 974
static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
				const struct g4x_wm_values *wm)
975
{
976 977 978 979 980
	enum pipe pipe;

	for_each_pipe(dev_priv, pipe)
		trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);

981
	intel_uncore_write(&dev_priv->uncore, DSPFW1,
982 983 984 985
		   FW_WM(wm->sr.plane, SR) |
		   FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
		   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
		   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
986
	intel_uncore_write(&dev_priv->uncore, DSPFW2,
987 988 989 990 991 992
		   (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
		   FW_WM(wm->sr.fbc, FBC_SR) |
		   FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
		   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
		   FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
		   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
993
	intel_uncore_write(&dev_priv->uncore, DSPFW3,
994 995 996 997
		   (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
		   FW_WM(wm->sr.cursor, CURSOR_SR) |
		   FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
		   FW_WM(wm->hpll.plane, HPLL_SR));
998

999
	intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1000 1001
}

1002 1003 1004
#define FW_WM_VLV(value, plane) \
	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)

1005
static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
1006 1007
				const struct vlv_wm_values *wm)
{
1008 1009 1010
	enum pipe pipe;

	for_each_pipe(dev_priv, pipe) {
1011 1012
		trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);

1013
		intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
1014 1015 1016 1017 1018
			   (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
			   (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
			   (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
			   (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
	}
1019

1020 1021 1022 1023 1024
	/*
	 * Zero the (unused) WM1 watermarks, and also clear all the
	 * high order bits so that there are no out of bounds values
	 * present in the registers during the reprogramming.
	 */
1025 1026 1027 1028 1029
	intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
	intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
	intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
	intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
	intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
1030

1031
	intel_uncore_write(&dev_priv->uncore, DSPFW1,
1032
		   FW_WM(wm->sr.plane, SR) |
1033 1034 1035
		   FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
		   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
		   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1036
	intel_uncore_write(&dev_priv->uncore, DSPFW2,
1037 1038 1039
		   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
		   FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
		   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1040
	intel_uncore_write(&dev_priv->uncore, DSPFW3,
1041
		   FW_WM(wm->sr.cursor, CURSOR_SR));
1042 1043

	if (IS_CHERRYVIEW(dev_priv)) {
1044
		intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1045 1046
			   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
			   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1047
		intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1048 1049
			   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
			   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1050
		intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1051 1052
			   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
			   FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1053
		intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1054
			   FW_WM(wm->sr.plane >> 9, SR_HI) |
1055 1056 1057 1058 1059 1060 1061 1062 1063
			   FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
			   FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
			   FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1064
	} else {
1065
		intel_uncore_write(&dev_priv->uncore, DSPFW7,
1066 1067
			   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
			   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1068
		intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1069
			   FW_WM(wm->sr.plane >> 9, SR_HI) |
1070 1071 1072 1073 1074 1075
			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1076 1077
	}

1078
	intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1079 1080
}

1081 1082
#undef FW_WM_VLV

1083 1084 1085 1086 1087
static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
{
	/* all latencies in usec */
	dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
	dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1088
	dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1089

1090
	dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134
}

static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
{
	/*
	 * DSPCNTR[13] supposedly controls whether the
	 * primary plane can use the FIFO space otherwise
	 * reserved for the sprite plane. It's not 100% clear
	 * what the actual FIFO size is, but it looks like we
	 * can happily set both primary and sprite watermarks
	 * up to 127 cachelines. So that would seem to mean
	 * that either DSPCNTR[13] doesn't do anything, or that
	 * the total FIFO is >= 256 cachelines in size. Either
	 * way, we don't seem to have to worry about this
	 * repartitioning as the maximum watermark value the
	 * register can hold for each plane is lower than the
	 * minimum FIFO size.
	 */
	switch (plane_id) {
	case PLANE_CURSOR:
		return 63;
	case PLANE_PRIMARY:
		return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
	case PLANE_SPRITE0:
		return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
	default:
		MISSING_CASE(plane_id);
		return 0;
	}
}

static int g4x_fbc_fifo_size(int level)
{
	switch (level) {
	case G4X_WM_LEVEL_SR:
		return 7;
	case G4X_WM_LEVEL_HPLL:
		return 15;
	default:
		MISSING_CASE(level);
		return 0;
	}
}

1135 1136 1137
static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
			  const struct intel_plane_state *plane_state,
			  int level)
1138
{
1139
	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1140
	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1141 1142
	const struct drm_display_mode *pipe_mode =
		&crtc_state->hw.pipe_mode;
1143 1144
	unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
	unsigned int clock, htotal, cpp, width, wm;
1145 1146 1147 1148 1149 1150 1151

	if (latency == 0)
		return USHRT_MAX;

	if (!intel_wm_plane_visible(crtc_state, plane_state))
		return 0;

1152
	cpp = plane_state->hw.fb->format->cpp[0];
1153

1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
	/*
	 * Not 100% sure which way ELK should go here as the
	 * spec only says CL/CTG should assume 32bpp and BW
	 * doesn't need to. But as these things followed the
	 * mobile vs. desktop lines on gen3 as well, let's
	 * assume ELK doesn't need this.
	 *
	 * The spec also fails to list such a restriction for
	 * the HPLL watermark, which seems a little strange.
	 * Let's use 32bpp for the HPLL watermark as well.
	 */
	if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
	    level != G4X_WM_LEVEL_NORMAL)
1167
		cpp = max(cpp, 4u);
1168

1169 1170
	clock = pipe_mode->crtc_clock;
	htotal = pipe_mode->crtc_htotal;
1171

1172
	width = drm_rect_width(&plane_state->uapi.dst);
1173 1174 1175 1176 1177 1178 1179

	if (plane->id == PLANE_CURSOR) {
		wm = intel_wm_method2(clock, htotal, width, cpp, latency);
	} else if (plane->id == PLANE_PRIMARY &&
		   level == G4X_WM_LEVEL_NORMAL) {
		wm = intel_wm_method1(clock, cpp, latency);
	} else {
1180
		unsigned int small, large;
1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192

		small = intel_wm_method1(clock, cpp, latency);
		large = intel_wm_method2(clock, htotal, width, cpp, latency);

		wm = min(small, large);
	}

	wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
			      width, cpp);

	wm = DIV_ROUND_UP(wm, 64) + 2;

1193
	return min_t(unsigned int, wm, USHRT_MAX);
1194 1195 1196 1197 1198
}

static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
				 int level, enum plane_id plane_id, u16 value)
{
1199
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214
	bool dirty = false;

	for (; level < intel_wm_num_levels(dev_priv); level++) {
		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];

		dirty |= raw->plane[plane_id] != value;
		raw->plane[plane_id] = value;
	}

	return dirty;
}

static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
			       int level, u16 value)
{
1215
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230
	bool dirty = false;

	/* NORMAL level doesn't have an FBC watermark */
	level = max(level, G4X_WM_LEVEL_SR);

	for (; level < intel_wm_num_levels(dev_priv); level++) {
		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];

		dirty |= raw->fbc != value;
		raw->fbc = value;
	}

	return dirty;
}

1231 1232
static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
			      const struct intel_plane_state *plane_state,
1233
			      u32 pri_val);
1234 1235 1236 1237

static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
				     const struct intel_plane_state *plane_state)
{
1238
	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1239
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291
	int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
	enum plane_id plane_id = plane->id;
	bool dirty = false;
	int level;

	if (!intel_wm_plane_visible(crtc_state, plane_state)) {
		dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
		if (plane_id == PLANE_PRIMARY)
			dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
		goto out;
	}

	for (level = 0; level < num_levels; level++) {
		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
		int wm, max_wm;

		wm = g4x_compute_wm(crtc_state, plane_state, level);
		max_wm = g4x_plane_fifo_size(plane_id, level);

		if (wm > max_wm)
			break;

		dirty |= raw->plane[plane_id] != wm;
		raw->plane[plane_id] = wm;

		if (plane_id != PLANE_PRIMARY ||
		    level == G4X_WM_LEVEL_NORMAL)
			continue;

		wm = ilk_compute_fbc_wm(crtc_state, plane_state,
					raw->plane[plane_id]);
		max_wm = g4x_fbc_fifo_size(level);

		/*
		 * FBC wm is not mandatory as we
		 * can always just disable its use.
		 */
		if (wm > max_wm)
			wm = USHRT_MAX;

		dirty |= raw->fbc != wm;
		raw->fbc = wm;
	}

	/* mark watermarks as invalid */
	dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);

	if (plane_id == PLANE_PRIMARY)
		dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);

 out:
	if (dirty) {
1292 1293 1294 1295 1296 1297
		drm_dbg_kms(&dev_priv->drm,
			    "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
			    plane->base.name,
			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1298 1299

		if (plane_id == PLANE_PRIMARY)
1300 1301 1302 1303
			drm_dbg_kms(&dev_priv->drm,
				    "FBC watermarks: SR=%d, HPLL=%d\n",
				    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
				    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319
	}

	return dirty;
}

static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
				      enum plane_id plane_id, int level)
{
	const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];

	return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
}

static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
				     int level)
{
1320
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355

	if (level > dev_priv->wm.max_level)
		return false;

	return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
		g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
		g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
}

/* mark all levels starting from 'level' as invalid */
static void g4x_invalidate_wms(struct intel_crtc *crtc,
			       struct g4x_wm_state *wm_state, int level)
{
	if (level <= G4X_WM_LEVEL_NORMAL) {
		enum plane_id plane_id;

		for_each_plane_id_on_crtc(crtc, plane_id)
			wm_state->wm.plane[plane_id] = USHRT_MAX;
	}

	if (level <= G4X_WM_LEVEL_SR) {
		wm_state->cxsr = false;
		wm_state->sr.cursor = USHRT_MAX;
		wm_state->sr.plane = USHRT_MAX;
		wm_state->sr.fbc = USHRT_MAX;
	}

	if (level <= G4X_WM_LEVEL_HPLL) {
		wm_state->hpll_en = false;
		wm_state->hpll.cursor = USHRT_MAX;
		wm_state->hpll.plane = USHRT_MAX;
		wm_state->hpll.fbc = USHRT_MAX;
	}
}

1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372
static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
			       int level)
{
	if (level < G4X_WM_LEVEL_SR)
		return false;

	if (level >= G4X_WM_LEVEL_SR &&
	    wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
		return false;

	if (level >= G4X_WM_LEVEL_HPLL &&
	    wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
		return false;

	return true;
}

1373 1374
static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
			       struct intel_crtc *crtc)
1375
{
1376 1377
	struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
1378
	struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1379 1380
	int num_active_planes = hweight8(crtc_state->active_planes &
					 ~BIT(PLANE_CURSOR));
1381
	const struct g4x_pipe_wm *raw;
1382 1383
	const struct intel_plane_state *old_plane_state;
	const struct intel_plane_state *new_plane_state;
1384 1385 1386 1387 1388
	struct intel_plane *plane;
	enum plane_id plane_id;
	int i, level;
	unsigned int dirty = 0;

1389 1390 1391
	for_each_oldnew_intel_plane_in_state(state, plane,
					     old_plane_state,
					     new_plane_state, i) {
1392 1393
		if (new_plane_state->hw.crtc != &crtc->base &&
		    old_plane_state->hw.crtc != &crtc->base)
1394 1395
			continue;

1396
		if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444
			dirty |= BIT(plane->id);
	}

	if (!dirty)
		return 0;

	level = G4X_WM_LEVEL_NORMAL;
	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
		goto out;

	raw = &crtc_state->wm.g4x.raw[level];
	for_each_plane_id_on_crtc(crtc, plane_id)
		wm_state->wm.plane[plane_id] = raw->plane[plane_id];

	level = G4X_WM_LEVEL_SR;
	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
		goto out;

	raw = &crtc_state->wm.g4x.raw[level];
	wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
	wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
	wm_state->sr.fbc = raw->fbc;

	wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);

	level = G4X_WM_LEVEL_HPLL;
	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
		goto out;

	raw = &crtc_state->wm.g4x.raw[level];
	wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
	wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
	wm_state->hpll.fbc = raw->fbc;

	wm_state->hpll_en = wm_state->cxsr;

	level++;

 out:
	if (level == G4X_WM_LEVEL_NORMAL)
		return -EINVAL;

	/* invalidate the higher levels */
	g4x_invalidate_wms(crtc, wm_state, level);

	/*
	 * Determine if the FBC watermark(s) can be used. IF
	 * this isn't the case we prefer to disable the FBC
1445 1446 1447
	 * watermark(s) rather than disable the SR/HPLL
	 * level(s) entirely. 'level-1' is the highest valid
	 * level here.
1448
	 */
1449
	wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1450 1451 1452 1453

	return 0;
}

1454 1455
static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
				       struct intel_crtc *crtc)
1456
{
1457
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1458 1459 1460 1461
	struct intel_crtc_state *new_crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
	const struct intel_crtc_state *old_crtc_state =
		intel_atomic_get_old_crtc_state(state, crtc);
1462 1463 1464
	struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
	const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
	const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1465 1466
	enum plane_id plane_id;

1467 1468
	if (!new_crtc_state->hw.active ||
	    drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1469 1470 1471 1472 1473 1474 1475
		*intermediate = *optimal;

		intermediate->cxsr = false;
		intermediate->hpll_en = false;
		goto out;
	}

1476
	intermediate->cxsr = optimal->cxsr && active->cxsr &&
1477
		!new_crtc_state->disable_cxsr;
1478
	intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1479
		!new_crtc_state->disable_cxsr;
1480 1481 1482 1483 1484 1485 1486
	intermediate->fbc_en = optimal->fbc_en && active->fbc_en;

	for_each_plane_id_on_crtc(crtc, plane_id) {
		intermediate->wm.plane[plane_id] =
			max(optimal->wm.plane[plane_id],
			    active->wm.plane[plane_id]);

1487 1488
		drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
			    g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504
	}

	intermediate->sr.plane = max(optimal->sr.plane,
				     active->sr.plane);
	intermediate->sr.cursor = max(optimal->sr.cursor,
				      active->sr.cursor);
	intermediate->sr.fbc = max(optimal->sr.fbc,
				   active->sr.fbc);

	intermediate->hpll.plane = max(optimal->hpll.plane,
				       active->hpll.plane);
	intermediate->hpll.cursor = max(optimal->hpll.cursor,
					active->hpll.cursor);
	intermediate->hpll.fbc = max(optimal->hpll.fbc,
				     active->hpll.fbc);

1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523
	drm_WARN_ON(&dev_priv->drm,
		    (intermediate->sr.plane >
		     g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
		     intermediate->sr.cursor >
		     g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
		    intermediate->cxsr);
	drm_WARN_ON(&dev_priv->drm,
		    (intermediate->sr.plane >
		     g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
		     intermediate->sr.cursor >
		     g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
		    intermediate->hpll_en);

	drm_WARN_ON(&dev_priv->drm,
		    intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
		    intermediate->fbc_en && intermediate->cxsr);
	drm_WARN_ON(&dev_priv->drm,
		    intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
		    intermediate->fbc_en && intermediate->hpll_en);
1524

1525
out:
1526 1527 1528 1529 1530
	/*
	 * If our intermediate WM are identical to the final WM, then we can
	 * omit the post-vblank programming; only update if it's different.
	 */
	if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1531
		new_crtc_state->wm.need_postvbl_update = true;
1532 1533 1534 1535 1536 1537 1538 1539

	return 0;
}

static void g4x_merge_wm(struct drm_i915_private *dev_priv,
			 struct g4x_wm_values *wm)
{
	struct intel_crtc *crtc;
1540
	int num_active_pipes = 0;
1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558

	wm->cxsr = true;
	wm->hpll_en = true;
	wm->fbc_en = true;

	for_each_intel_crtc(&dev_priv->drm, crtc) {
		const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;

		if (!crtc->active)
			continue;

		if (!wm_state->cxsr)
			wm->cxsr = false;
		if (!wm_state->hpll_en)
			wm->hpll_en = false;
		if (!wm_state->fbc_en)
			wm->fbc_en = false;

1559
		num_active_pipes++;
1560 1561
	}

1562
	if (num_active_pipes != 1) {
1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601
		wm->cxsr = false;
		wm->hpll_en = false;
		wm->fbc_en = false;
	}

	for_each_intel_crtc(&dev_priv->drm, crtc) {
		const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
		enum pipe pipe = crtc->pipe;

		wm->pipe[pipe] = wm_state->wm;
		if (crtc->active && wm->cxsr)
			wm->sr = wm_state->sr;
		if (crtc->active && wm->hpll_en)
			wm->hpll = wm_state->hpll;
	}
}

static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
{
	struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
	struct g4x_wm_values new_wm = {};

	g4x_merge_wm(dev_priv, &new_wm);

	if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
		return;

	if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
		_intel_set_memory_cxsr(dev_priv, false);

	g4x_write_wm_values(dev_priv, &new_wm);

	if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
		_intel_set_memory_cxsr(dev_priv, true);

	*old_wm = new_wm;
}

static void g4x_initial_watermarks(struct intel_atomic_state *state,
1602
				   struct intel_crtc *crtc)
1603
{
1604 1605 1606
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	const struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
1607 1608 1609 1610 1611 1612 1613 1614

	mutex_lock(&dev_priv->wm.wm_mutex);
	crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
	g4x_program_watermarks(dev_priv);
	mutex_unlock(&dev_priv->wm.wm_mutex);
}

static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1615
				    struct intel_crtc *crtc)
1616
{
1617 1618 1619
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	const struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
1620 1621 1622 1623 1624

	if (!crtc_state->wm.need_postvbl_update)
		return;

	mutex_lock(&dev_priv->wm.wm_mutex);
1625
	crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1626 1627 1628 1629
	g4x_program_watermarks(dev_priv);
	mutex_unlock(&dev_priv->wm.wm_mutex);
}

1630 1631
/* latency must be in 0.1us units. */
static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1632 1633
				   unsigned int htotal,
				   unsigned int width,
1634
				   unsigned int cpp,
1635 1636 1637 1638
				   unsigned int latency)
{
	unsigned int ret;

1639 1640
	ret = intel_wm_method2(pixel_rate, htotal,
			       width, cpp, latency);
1641 1642 1643 1644 1645
	ret = DIV_ROUND_UP(ret, 64);

	return ret;
}

1646
static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1647 1648 1649 1650
{
	/* all latencies in usec */
	dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;

1651 1652
	dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;

1653 1654 1655
	if (IS_CHERRYVIEW(dev_priv)) {
		dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
		dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1656 1657

		dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1658 1659 1660
	}
}

1661 1662 1663
static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
				const struct intel_plane_state *plane_state,
				int level)
1664
{
1665
	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1666
	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1667 1668
	const struct drm_display_mode *pipe_mode =
		&crtc_state->hw.pipe_mode;
1669
	unsigned int clock, htotal, cpp, width, wm;
1670 1671 1672 1673

	if (dev_priv->wm.pri_latency[level] == 0)
		return USHRT_MAX;

1674
	if (!intel_wm_plane_visible(crtc_state, plane_state))
1675 1676
		return 0;

1677
	cpp = plane_state->hw.fb->format->cpp[0];
1678 1679
	clock = pipe_mode->crtc_clock;
	htotal = pipe_mode->crtc_htotal;
1680
	width = crtc_state->pipe_src_w;
1681

1682
	if (plane->id == PLANE_CURSOR) {
1683 1684 1685 1686 1687 1688 1689 1690
		/*
		 * FIXME the formula gives values that are
		 * too big for the cursor FIFO, and hence we
		 * would never be able to use cursors. For
		 * now just hardcode the watermark.
		 */
		wm = 63;
	} else {
1691
		wm = vlv_wm_method2(clock, htotal, width, cpp,
1692 1693 1694
				    dev_priv->wm.pri_latency[level] * 10);
	}

1695
	return min_t(unsigned int, wm, USHRT_MAX);
1696 1697
}

1698 1699 1700 1701 1702 1703
static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
{
	return (active_planes & (BIT(PLANE_SPRITE0) |
				 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
}

1704
static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1705
{
1706
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1707
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1708
	const struct g4x_pipe_wm *raw =
1709
		&crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1710
	struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1711
	unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1712
	int num_active_planes = hweight8(active_planes);
1713
	const int fifo_size = 511;
1714
	int fifo_extra, fifo_left = fifo_size;
1715
	int sprite0_fifo_extra = 0;
1716 1717
	unsigned int total_rate;
	enum plane_id plane_id;
1718

1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729
	/*
	 * When enabling sprite0 after sprite1 has already been enabled
	 * we tend to get an underrun unless sprite0 already has some
	 * FIFO space allcoated. Hence we always allocate at least one
	 * cacheline for sprite0 whenever sprite1 is enabled.
	 *
	 * All other plane enable sequences appear immune to this problem.
	 */
	if (vlv_need_sprite0_fifo_workaround(active_planes))
		sprite0_fifo_extra = 1;

1730 1731
	total_rate = raw->plane[PLANE_PRIMARY] +
		raw->plane[PLANE_SPRITE0] +
1732 1733
		raw->plane[PLANE_SPRITE1] +
		sprite0_fifo_extra;
1734

1735 1736
	if (total_rate > fifo_size)
		return -EINVAL;
1737

1738 1739
	if (total_rate == 0)
		total_rate = 1;
1740

1741
	for_each_plane_id_on_crtc(crtc, plane_id) {
1742 1743
		unsigned int rate;

1744 1745
		if ((active_planes & BIT(plane_id)) == 0) {
			fifo_state->plane[plane_id] = 0;
1746 1747 1748
			continue;
		}

1749 1750 1751
		rate = raw->plane[plane_id];
		fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
		fifo_left -= fifo_state->plane[plane_id];
1752 1753
	}

1754 1755 1756
	fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
	fifo_left -= sprite0_fifo_extra;

1757 1758 1759
	fifo_state->plane[PLANE_CURSOR] = 63;

	fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1760 1761

	/* spread the remainder evenly */
1762
	for_each_plane_id_on_crtc(crtc, plane_id) {
1763 1764 1765 1766 1767
		int plane_extra;

		if (fifo_left == 0)
			break;

1768
		if ((active_planes & BIT(plane_id)) == 0)
1769 1770 1771
			continue;

		plane_extra = min(fifo_extra, fifo_left);
1772
		fifo_state->plane[plane_id] += plane_extra;
1773 1774 1775
		fifo_left -= plane_extra;
	}

1776
	drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1777 1778 1779

	/* give it all to the first plane if none are active */
	if (active_planes == 0) {
1780
		drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1781 1782 1783 1784
		fifo_state->plane[PLANE_PRIMARY] = fifo_left;
	}

	return 0;
1785 1786
}

1787 1788 1789 1790 1791 1792
/* mark all levels starting from 'level' as invalid */
static void vlv_invalidate_wms(struct intel_crtc *crtc,
			       struct vlv_wm_state *wm_state, int level)
{
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);

1793
	for (; level < intel_wm_num_levels(dev_priv); level++) {
1794 1795 1796 1797 1798 1799 1800 1801 1802 1803
		enum plane_id plane_id;

		for_each_plane_id_on_crtc(crtc, plane_id)
			wm_state->wm[level].plane[plane_id] = USHRT_MAX;

		wm_state->sr[level].cursor = USHRT_MAX;
		wm_state->sr[level].plane = USHRT_MAX;
	}
}

1804 1805 1806 1807 1808 1809 1810 1811
static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
{
	if (wm > fifo_size)
		return USHRT_MAX;
	else
		return fifo_size - wm;
}

1812 1813 1814 1815
/*
 * Starting from 'level' set all higher
 * levels to 'value' in the "raw" watermarks.
 */
1816
static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1817
				 int level, enum plane_id plane_id, u16 value)
1818
{
1819
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1820
	int num_levels = intel_wm_num_levels(dev_priv);
1821
	bool dirty = false;
1822

1823
	for (; level < num_levels; level++) {
1824
		struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1825

1826
		dirty |= raw->plane[plane_id] != value;
1827
		raw->plane[plane_id] = value;
1828
	}
1829 1830

	return dirty;
1831 1832
}

1833 1834
static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
				     const struct intel_plane_state *plane_state)
1835
{
1836
	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1837
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1838
	enum plane_id plane_id = plane->id;
1839
	int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1840
	int level;
1841
	bool dirty = false;
1842

1843
	if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1844 1845
		dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
		goto out;
1846
	}
1847

1848
	for (level = 0; level < num_levels; level++) {
1849
		struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1850 1851
		int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
		int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1852

1853 1854
		if (wm > max_wm)
			break;
1855

1856
		dirty |= raw->plane[plane_id] != wm;
1857 1858
		raw->plane[plane_id] = wm;
	}
1859

1860
	/* mark all higher levels as invalid */
1861
	dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1862

1863 1864
out:
	if (dirty)
1865 1866 1867 1868 1869 1870
		drm_dbg_kms(&dev_priv->drm,
			    "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
			    plane->base.name,
			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1871 1872

	return dirty;
1873
}
1874

1875 1876
static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
				      enum plane_id plane_id, int level)
1877
{
1878
	const struct g4x_pipe_wm *raw =
1879 1880 1881
		&crtc_state->wm.vlv.raw[level];
	const struct vlv_fifo_state *fifo_state =
		&crtc_state->wm.vlv.fifo_state;
1882

1883 1884
	return raw->plane[plane_id] <= fifo_state->plane[plane_id];
}
1885

1886
static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1887
{
1888 1889 1890 1891
	return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1892 1893
}

1894 1895
static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
			       struct intel_crtc *crtc)
1896 1897
{
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1898 1899
	struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
1900 1901 1902
	struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
	const struct vlv_fifo_state *fifo_state =
		&crtc_state->wm.vlv.fifo_state;
1903 1904
	int num_active_planes = hweight8(crtc_state->active_planes &
					 ~BIT(PLANE_CURSOR));
1905
	bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1906 1907
	const struct intel_plane_state *old_plane_state;
	const struct intel_plane_state *new_plane_state;
1908 1909 1910
	struct intel_plane *plane;
	enum plane_id plane_id;
	int level, ret, i;
1911
	unsigned int dirty = 0;
1912

1913 1914 1915
	for_each_oldnew_intel_plane_in_state(state, plane,
					     old_plane_state,
					     new_plane_state, i) {
1916 1917
		if (new_plane_state->hw.crtc != &crtc->base &&
		    old_plane_state->hw.crtc != &crtc->base)
1918
			continue;
1919

1920
		if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938
			dirty |= BIT(plane->id);
	}

	/*
	 * DSPARB registers may have been reset due to the
	 * power well being turned off. Make sure we restore
	 * them to a consistent state even if no primary/sprite
	 * planes are initially active.
	 */
	if (needs_modeset)
		crtc_state->fifo_changed = true;

	if (!dirty)
		return 0;

	/* cursor changes don't warrant a FIFO recompute */
	if (dirty & ~BIT(PLANE_CURSOR)) {
		const struct intel_crtc_state *old_crtc_state =
1939
			intel_atomic_get_old_crtc_state(state, crtc);
1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950
		const struct vlv_fifo_state *old_fifo_state =
			&old_crtc_state->wm.vlv.fifo_state;

		ret = vlv_compute_fifo(crtc_state);
		if (ret)
			return ret;

		if (needs_modeset ||
		    memcmp(old_fifo_state, fifo_state,
			   sizeof(*fifo_state)) != 0)
			crtc_state->fifo_changed = true;
1951
	}
1952

1953
	/* initially allow all levels */
1954
	wm_state->num_levels = intel_wm_num_levels(dev_priv);
1955 1956 1957 1958 1959
	/*
	 * Note that enabling cxsr with no primary/sprite planes
	 * enabled can wedge the pipe. Hence we only allow cxsr
	 * with exactly one enabled primary/sprite plane.
	 */
1960
	wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1961

1962
	for (level = 0; level < wm_state->num_levels; level++) {
1963
		const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1964
		const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1965

1966
		if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1967
			break;
1968

1969 1970 1971 1972 1973 1974 1975 1976
		for_each_plane_id_on_crtc(crtc, plane_id) {
			wm_state->wm[level].plane[plane_id] =
				vlv_invert_wm_value(raw->plane[plane_id],
						    fifo_state->plane[plane_id]);
		}

		wm_state->sr[level].plane =
			vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1977
						 raw->plane[PLANE_SPRITE0],
1978 1979
						 raw->plane[PLANE_SPRITE1]),
					    sr_fifo_size);
1980

1981 1982 1983
		wm_state->sr[level].cursor =
			vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
					    63);
1984 1985
	}

1986 1987 1988 1989 1990 1991 1992 1993 1994 1995
	if (level == 0)
		return -EINVAL;

	/* limit to only levels we can actually handle */
	wm_state->num_levels = level;

	/* invalidate the higher levels */
	vlv_invalidate_wms(crtc, wm_state, level);

	return 0;
1996 1997
}

1998 1999 2000
#define VLV_FIFO(plane, value) \
	(((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)

2001
static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
2002
				   struct intel_crtc *crtc)
2003
{
2004
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2005
	struct intel_uncore *uncore = &dev_priv->uncore;
2006 2007
	const struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
2008 2009
	const struct vlv_fifo_state *fifo_state =
		&crtc_state->wm.vlv.fifo_state;
2010
	int sprite0_start, sprite1_start, fifo_size;
2011
	u32 dsparb, dsparb2, dsparb3;
2012

2013 2014 2015
	if (!crtc_state->fifo_changed)
		return;

2016 2017 2018
	sprite0_start = fifo_state->plane[PLANE_PRIMARY];
	sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
	fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2019

2020 2021
	drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
	drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2022

2023 2024
	trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);

2025 2026 2027 2028 2029 2030 2031 2032 2033
	/*
	 * uncore.lock serves a double purpose here. It allows us to
	 * use the less expensive I915_{READ,WRITE}_FW() functions, and
	 * it protects the DSPARB registers from getting clobbered by
	 * parallel updates from multiple pipes.
	 *
	 * intel_pipe_update_start() has already disabled interrupts
	 * for us, so a plain spin_lock() is sufficient here.
	 */
2034
	spin_lock(&uncore->lock);
2035

2036 2037
	switch (crtc->pipe) {
	case PIPE_A:
2038 2039
		dsparb = intel_uncore_read_fw(uncore, DSPARB);
		dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050

		dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
			    VLV_FIFO(SPRITEB, 0xff));
		dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
			   VLV_FIFO(SPRITEB, sprite1_start));

		dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
			     VLV_FIFO(SPRITEB_HI, 0x1));
		dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
			   VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));

2051 2052
		intel_uncore_write_fw(uncore, DSPARB, dsparb);
		intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2053 2054
		break;
	case PIPE_B:
2055 2056
		dsparb = intel_uncore_read_fw(uncore, DSPARB);
		dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067

		dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
			    VLV_FIFO(SPRITED, 0xff));
		dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
			   VLV_FIFO(SPRITED, sprite1_start));

		dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
			     VLV_FIFO(SPRITED_HI, 0xff));
		dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
			   VLV_FIFO(SPRITED_HI, sprite1_start >> 8));

2068 2069
		intel_uncore_write_fw(uncore, DSPARB, dsparb);
		intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2070 2071
		break;
	case PIPE_C:
2072 2073
		dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
		dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084

		dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
			     VLV_FIFO(SPRITEF, 0xff));
		dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
			    VLV_FIFO(SPRITEF, sprite1_start));

		dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
			     VLV_FIFO(SPRITEF_HI, 0xff));
		dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
			   VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));

2085 2086
		intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
		intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2087 2088 2089 2090
		break;
	default:
		break;
	}
2091

2092
	intel_uncore_posting_read_fw(uncore, DSPARB);
2093

2094
	spin_unlock(&uncore->lock);
2095 2096 2097 2098
}

#undef VLV_FIFO

2099 2100
static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
				       struct intel_crtc *crtc)
2101
{
2102 2103 2104 2105
	struct intel_crtc_state *new_crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
	const struct intel_crtc_state *old_crtc_state =
		intel_atomic_get_old_crtc_state(state, crtc);
2106 2107 2108
	struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
	const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
	const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2109 2110
	int level;

2111 2112
	if (!new_crtc_state->hw.active ||
	    drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2113 2114 2115 2116 2117 2118
		*intermediate = *optimal;

		intermediate->cxsr = false;
		goto out;
	}

2119
	intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2120
	intermediate->cxsr = optimal->cxsr && active->cxsr &&
2121
		!new_crtc_state->disable_cxsr;
2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139

	for (level = 0; level < intermediate->num_levels; level++) {
		enum plane_id plane_id;

		for_each_plane_id_on_crtc(crtc, plane_id) {
			intermediate->wm[level].plane[plane_id] =
				min(optimal->wm[level].plane[plane_id],
				    active->wm[level].plane[plane_id]);
		}

		intermediate->sr[level].plane = min(optimal->sr[level].plane,
						    active->sr[level].plane);
		intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
						     active->sr[level].cursor);
	}

	vlv_invalidate_wms(crtc, intermediate, level);

2140
out:
2141 2142 2143 2144
	/*
	 * If our intermediate WM are identical to the final WM, then we can
	 * omit the post-vblank programming; only update if it's different.
	 */
2145
	if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2146
		new_crtc_state->wm.need_postvbl_update = true;
2147 2148 2149 2150

	return 0;
}

2151
static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2152 2153 2154
			 struct vlv_wm_values *wm)
{
	struct intel_crtc *crtc;
2155
	int num_active_pipes = 0;
2156

2157
	wm->level = dev_priv->wm.max_level;
2158 2159
	wm->cxsr = true;

2160
	for_each_intel_crtc(&dev_priv->drm, crtc) {
2161
		const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2162 2163 2164 2165 2166 2167 2168

		if (!crtc->active)
			continue;

		if (!wm_state->cxsr)
			wm->cxsr = false;

2169
		num_active_pipes++;
2170 2171 2172
		wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
	}

2173
	if (num_active_pipes != 1)
2174 2175
		wm->cxsr = false;

2176
	if (num_active_pipes > 1)
2177 2178
		wm->level = VLV_WM_LEVEL_PM2;

2179
	for_each_intel_crtc(&dev_priv->drm, crtc) {
2180
		const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2181 2182 2183
		enum pipe pipe = crtc->pipe;

		wm->pipe[pipe] = wm_state->wm[wm->level];
2184
		if (crtc->active && wm->cxsr)
2185 2186
			wm->sr = wm_state->sr[wm->level];

2187 2188 2189 2190
		wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
		wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
		wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
		wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2191 2192 2193
	}
}

2194
static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2195
{
2196 2197
	struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
	struct vlv_wm_values new_wm = {};
2198

2199
	vlv_merge_wm(dev_priv, &new_wm);
2200

2201
	if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2202 2203
		return;

2204
	if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2205 2206
		chv_set_memory_dvfs(dev_priv, false);

2207
	if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2208 2209
		chv_set_memory_pm5(dev_priv, false);

2210
	if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2211
		_intel_set_memory_cxsr(dev_priv, false);
2212

2213
	vlv_write_wm_values(dev_priv, &new_wm);
2214

2215
	if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2216
		_intel_set_memory_cxsr(dev_priv, true);
2217

2218
	if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2219 2220
		chv_set_memory_pm5(dev_priv, true);

2221
	if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2222 2223
		chv_set_memory_dvfs(dev_priv, true);

2224
	*old_wm = new_wm;
2225 2226
}

2227
static void vlv_initial_watermarks(struct intel_atomic_state *state,
2228
				   struct intel_crtc *crtc)
2229
{
2230 2231 2232
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	const struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
2233 2234

	mutex_lock(&dev_priv->wm.wm_mutex);
2235 2236 2237 2238 2239 2240
	crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
	vlv_program_watermarks(dev_priv);
	mutex_unlock(&dev_priv->wm.wm_mutex);
}

static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2241
				    struct intel_crtc *crtc)
2242
{
2243 2244 2245
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	const struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
2246 2247 2248 2249 2250

	if (!crtc_state->wm.need_postvbl_update)
		return;

	mutex_lock(&dev_priv->wm.wm_mutex);
2251
	crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2252 2253 2254 2255
	vlv_program_watermarks(dev_priv);
	mutex_unlock(&dev_priv->wm.wm_mutex);
}

2256
static void i965_update_wm(struct intel_crtc *unused_crtc)
2257
{
2258
	struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2259
	struct intel_crtc *crtc;
2260 2261
	int srwm = 1;
	int cursor_sr = 16;
2262
	bool cxsr_enabled;
2263 2264

	/* Calc sr entries for one plane configs */
2265
	crtc = single_enabled_crtc(dev_priv);
2266 2267 2268
	if (crtc) {
		/* self-refresh has much higher latency */
		static const int sr_latency_ns = 12000;
2269 2270
		const struct drm_display_mode *pipe_mode =
			&crtc->config->hw.pipe_mode;
2271 2272
		const struct drm_framebuffer *fb =
			crtc->base.primary->state->fb;
2273 2274
		int clock = pipe_mode->crtc_clock;
		int htotal = pipe_mode->crtc_htotal;
2275
		int hdisplay = crtc->config->pipe_src_w;
2276
		int cpp = fb->format->cpp[0];
2277 2278
		int entries;

2279 2280
		entries = intel_wm_method2(clock, htotal,
					   hdisplay, cpp, sr_latency_ns / 100);
2281 2282 2283 2284 2285
		entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		srwm = I965_FIFO_SIZE - entries;
		if (srwm < 0)
			srwm = 1;
		srwm &= 0x1ff;
2286 2287 2288
		drm_dbg_kms(&dev_priv->drm,
			    "self-refresh entries: %d, wm: %d\n",
			    entries, srwm);
2289

2290 2291 2292
		entries = intel_wm_method2(clock, htotal,
					   crtc->base.cursor->state->crtc_w, 4,
					   sr_latency_ns / 100);
2293
		entries = DIV_ROUND_UP(entries,
2294 2295
				       i965_cursor_wm_info.cacheline_size) +
			i965_cursor_wm_info.guard_size;
2296

2297
		cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2298 2299 2300
		if (cursor_sr > i965_cursor_wm_info.max_wm)
			cursor_sr = i965_cursor_wm_info.max_wm;

2301 2302 2303
		drm_dbg_kms(&dev_priv->drm,
			    "self-refresh watermark: display plane %d "
			    "cursor %d\n", srwm, cursor_sr);
2304

2305
		cxsr_enabled = true;
2306
	} else {
2307
		cxsr_enabled = false;
2308
		/* Turn off self refresh if both pipes are enabled */
2309
		intel_set_memory_cxsr(dev_priv, false);
2310 2311
	}

2312 2313 2314
	drm_dbg_kms(&dev_priv->drm,
		    "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		    srwm);
2315 2316

	/* 965 has limitations... */
2317
	intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2318 2319 2320
		   FW_WM(8, CURSORB) |
		   FW_WM(8, PLANEB) |
		   FW_WM(8, PLANEA));
2321
	intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2322
		   FW_WM(8, PLANEC_OLD));
2323
	/* update cursor SR watermark */
2324
	intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2325 2326 2327

	if (cxsr_enabled)
		intel_set_memory_cxsr(dev_priv, true);
2328 2329
}

2330 2331
#undef FW_WM

2332
static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2333
{
2334
	struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2335
	const struct intel_watermark_params *wm_info;
2336 2337
	u32 fwater_lo;
	u32 fwater_hi;
2338 2339 2340
	int cwm, srwm = 1;
	int fifo_size;
	int planea_wm, planeb_wm;
2341
	struct intel_crtc *crtc, *enabled = NULL;
2342

2343
	if (IS_I945GM(dev_priv))
2344
		wm_info = &i945_wm_info;
2345
	else if (DISPLAY_VER(dev_priv) != 2)
2346 2347
		wm_info = &i915_wm_info;
	else
2348
		wm_info = &i830_a_wm_info;
2349

2350 2351
	fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
	crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2352
	if (intel_crtc_active(crtc)) {
2353 2354
		const struct drm_display_mode *pipe_mode =
			&crtc->config->hw.pipe_mode;
2355 2356 2357 2358
		const struct drm_framebuffer *fb =
			crtc->base.primary->state->fb;
		int cpp;

2359
		if (DISPLAY_VER(dev_priv) == 2)
2360
			cpp = 4;
2361
		else
2362
			cpp = fb->format->cpp[0];
2363

2364
		planea_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2365
					       wm_info, fifo_size, cpp,
2366
					       pessimal_latency_ns);
2367
		enabled = crtc;
2368
	} else {
2369
		planea_wm = fifo_size - wm_info->guard_size;
2370 2371 2372 2373
		if (planea_wm > (long)wm_info->max_wm)
			planea_wm = wm_info->max_wm;
	}

2374
	if (DISPLAY_VER(dev_priv) == 2)
2375
		wm_info = &i830_bc_wm_info;
2376

2377 2378
	fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
	crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2379
	if (intel_crtc_active(crtc)) {
2380 2381
		const struct drm_display_mode *pipe_mode =
			&crtc->config->hw.pipe_mode;
2382 2383 2384 2385
		const struct drm_framebuffer *fb =
			crtc->base.primary->state->fb;
		int cpp;

2386
		if (DISPLAY_VER(dev_priv) == 2)
2387
			cpp = 4;
2388
		else
2389
			cpp = fb->format->cpp[0];
2390

2391
		planeb_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2392
					       wm_info, fifo_size, cpp,
2393
					       pessimal_latency_ns);
2394 2395 2396 2397
		if (enabled == NULL)
			enabled = crtc;
		else
			enabled = NULL;
2398
	} else {
2399
		planeb_wm = fifo_size - wm_info->guard_size;
2400 2401 2402
		if (planeb_wm > (long)wm_info->max_wm)
			planeb_wm = wm_info->max_wm;
	}
2403

2404 2405
	drm_dbg_kms(&dev_priv->drm,
		    "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2406

2407
	if (IS_I915GM(dev_priv) && enabled) {
2408
		struct drm_i915_gem_object *obj;
2409

2410
		obj = intel_fb_obj(enabled->base.primary->state->fb);
2411 2412

		/* self-refresh seems busted with untiled */
2413
		if (!i915_gem_object_is_tiled(obj))
2414 2415 2416
			enabled = NULL;
	}

2417 2418 2419 2420 2421 2422
	/*
	 * Overlay gets an aggressive default since video jitter is bad.
	 */
	cwm = 2;

	/* Play safe and disable self-refresh before adjusting watermarks. */
2423
	intel_set_memory_cxsr(dev_priv, false);
2424 2425

	/* Calc sr entries for one plane configs */
2426
	if (HAS_FW_BLC(dev_priv) && enabled) {
2427 2428
		/* self-refresh has much higher latency */
		static const int sr_latency_ns = 6000;
2429 2430
		const struct drm_display_mode *pipe_mode =
			&enabled->config->hw.pipe_mode;
2431 2432
		const struct drm_framebuffer *fb =
			enabled->base.primary->state->fb;
2433 2434
		int clock = pipe_mode->crtc_clock;
		int htotal = pipe_mode->crtc_htotal;
2435 2436
		int hdisplay = enabled->config->pipe_src_w;
		int cpp;
2437 2438
		int entries;

2439
		if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2440
			cpp = 4;
2441
		else
2442
			cpp = fb->format->cpp[0];
2443

2444 2445
		entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
					   sr_latency_ns / 100);
2446
		entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2447 2448
		drm_dbg_kms(&dev_priv->drm,
			    "self-refresh entries: %d\n", entries);
2449 2450 2451 2452
		srwm = wm_info->fifo_size - entries;
		if (srwm < 0)
			srwm = 1;

2453
		if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2454
			intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2455
				   FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2456
		else
2457
			intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2458 2459
	}

2460 2461 2462
	drm_dbg_kms(&dev_priv->drm,
		    "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		     planea_wm, planeb_wm, cwm, srwm);
2463 2464 2465 2466 2467 2468 2469 2470

	fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
	fwater_hi = (cwm & 0x1f);

	/* Set request length to 8 cachelines per fetch */
	fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
	fwater_hi = fwater_hi | (1 << 8);

2471 2472
	intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
	intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2473

2474 2475
	if (enabled)
		intel_set_memory_cxsr(dev_priv, true);
2476 2477
}

2478
static void i845_update_wm(struct intel_crtc *unused_crtc)
2479
{
2480
	struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2481
	struct intel_crtc *crtc;
2482
	const struct drm_display_mode *pipe_mode;
2483
	u32 fwater_lo;
2484 2485
	int planea_wm;

2486
	crtc = single_enabled_crtc(dev_priv);
2487 2488 2489
	if (crtc == NULL)
		return;

2490 2491
	pipe_mode = &crtc->config->hw.pipe_mode;
	planea_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2492
				       &i845_wm_info,
2493
				       dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2494
				       4, pessimal_latency_ns);
2495
	fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2496 2497
	fwater_lo |= (3<<8) | planea_wm;

2498 2499
	drm_dbg_kms(&dev_priv->drm,
		    "Setting FIFO watermarks - A: %d\n", planea_wm);
2500

2501
	intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2502 2503
}

2504
/* latency must be in 0.1us units. */
2505 2506 2507
static unsigned int ilk_wm_method1(unsigned int pixel_rate,
				   unsigned int cpp,
				   unsigned int latency)
2508
{
2509
	unsigned int ret;
2510

2511 2512
	ret = intel_wm_method1(pixel_rate, cpp, latency);
	ret = DIV_ROUND_UP(ret, 64) + 2;
2513 2514 2515 2516

	return ret;
}

2517
/* latency must be in 0.1us units. */
2518 2519 2520 2521 2522
static unsigned int ilk_wm_method2(unsigned int pixel_rate,
				   unsigned int htotal,
				   unsigned int width,
				   unsigned int cpp,
				   unsigned int latency)
2523
{
2524
	unsigned int ret;
2525

2526 2527
	ret = intel_wm_method2(pixel_rate, htotal,
			       width, cpp, latency);
2528
	ret = DIV_ROUND_UP(ret, 64) + 2;
2529

2530 2531 2532
	return ret;
}

2533
static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2534
{
2535 2536 2537 2538 2539 2540
	/*
	 * Neither of these should be possible since this function shouldn't be
	 * called if the CRTC is off or the plane is invisible.  But let's be
	 * extra paranoid to avoid a potential divide-by-zero if we screw up
	 * elsewhere in the driver.
	 */
2541
	if (WARN_ON(!cpp))
2542 2543 2544 2545
		return 0;
	if (WARN_ON(!horiz_pixels))
		return 0;

2546
	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2547 2548
}

2549
struct ilk_wm_maximums {
2550 2551 2552 2553
	u16 pri;
	u16 spr;
	u16 cur;
	u16 fbc;
2554 2555
};

2556 2557 2558 2559
/*
 * For both WM_PIPE and WM_LP.
 * mem_value must be in 0.1us units.
 */
2560 2561
static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
			      const struct intel_plane_state *plane_state,
2562
			      u32 mem_value, bool is_lp)
2563
{
2564
	u32 method1, method2;
2565
	int cpp;
2566

2567 2568 2569
	if (mem_value == 0)
		return U32_MAX;

2570
	if (!intel_wm_plane_visible(crtc_state, plane_state))
2571 2572
		return 0;

2573
	cpp = plane_state->hw.fb->format->cpp[0];
2574

2575
	method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2576 2577 2578 2579

	if (!is_lp)
		return method1;

2580
	method2 = ilk_wm_method2(crtc_state->pixel_rate,
2581
				 crtc_state->hw.pipe_mode.crtc_htotal,
2582
				 drm_rect_width(&plane_state->uapi.dst),
2583
				 cpp, mem_value);
2584 2585

	return min(method1, method2);
2586 2587
}

2588 2589 2590 2591
/*
 * For both WM_PIPE and WM_LP.
 * mem_value must be in 0.1us units.
 */
2592 2593
static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
			      const struct intel_plane_state *plane_state,
2594
			      u32 mem_value)
2595
{
2596
	u32 method1, method2;
2597
	int cpp;
2598

2599 2600 2601
	if (mem_value == 0)
		return U32_MAX;

2602
	if (!intel_wm_plane_visible(crtc_state, plane_state))
2603 2604
		return 0;

2605
	cpp = plane_state->hw.fb->format->cpp[0];
2606

2607 2608
	method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
	method2 = ilk_wm_method2(crtc_state->pixel_rate,
2609
				 crtc_state->hw.pipe_mode.crtc_htotal,
2610
				 drm_rect_width(&plane_state->uapi.dst),
2611
				 cpp, mem_value);
2612 2613 2614
	return min(method1, method2);
}

2615 2616 2617 2618
/*
 * For both WM_PIPE and WM_LP.
 * mem_value must be in 0.1us units.
 */
2619 2620
static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
			      const struct intel_plane_state *plane_state,
2621
			      u32 mem_value)
2622
{
2623 2624
	int cpp;

2625 2626 2627
	if (mem_value == 0)
		return U32_MAX;

2628
	if (!intel_wm_plane_visible(crtc_state, plane_state))
2629 2630
		return 0;

2631
	cpp = plane_state->hw.fb->format->cpp[0];
2632

2633
	return ilk_wm_method2(crtc_state->pixel_rate,
2634
			      crtc_state->hw.pipe_mode.crtc_htotal,
2635
			      drm_rect_width(&plane_state->uapi.dst),
2636
			      cpp, mem_value);
2637 2638
}

2639
/* Only for WM_LP. */
2640 2641
static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
			      const struct intel_plane_state *plane_state,
2642
			      u32 pri_val)
2643
{
2644
	int cpp;
2645

2646
	if (!intel_wm_plane_visible(crtc_state, plane_state))
2647 2648
		return 0;

2649
	cpp = plane_state->hw.fb->format->cpp[0];
2650

2651 2652
	return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
			  cpp);
2653 2654
}

2655 2656
static unsigned int
ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2657
{
2658
	if (DISPLAY_VER(dev_priv) >= 8)
2659
		return 3072;
2660
	else if (DISPLAY_VER(dev_priv) >= 7)
2661 2662 2663 2664 2665
		return 768;
	else
		return 512;
}

2666 2667 2668
static unsigned int
ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
		     int level, bool is_sprite)
2669
{
2670
	if (DISPLAY_VER(dev_priv) >= 8)
2671 2672
		/* BDW primary/sprite plane watermarks */
		return level == 0 ? 255 : 2047;
2673
	else if (DISPLAY_VER(dev_priv) >= 7)
2674 2675 2676 2677 2678 2679 2680 2681 2682 2683
		/* IVB/HSW primary/sprite plane watermarks */
		return level == 0 ? 127 : 1023;
	else if (!is_sprite)
		/* ILK/SNB primary plane watermarks */
		return level == 0 ? 127 : 511;
	else
		/* ILK/SNB sprite plane watermarks */
		return level == 0 ? 63 : 255;
}

2684 2685
static unsigned int
ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2686
{
2687
	if (DISPLAY_VER(dev_priv) >= 7)
2688 2689 2690 2691 2692
		return level == 0 ? 63 : 255;
	else
		return level == 0 ? 31 : 63;
}

2693
static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2694
{
2695
	if (DISPLAY_VER(dev_priv) >= 8)
2696 2697 2698 2699 2700
		return 31;
	else
		return 15;
}

2701
/* Calculate the maximum primary/sprite plane watermark */
2702
static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2703
				     int level,
2704
				     const struct intel_wm_config *config,
2705 2706 2707
				     enum intel_ddb_partitioning ddb_partitioning,
				     bool is_sprite)
{
2708
	unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2709 2710

	/* if sprites aren't enabled, sprites get nothing */
2711
	if (is_sprite && !config->sprites_enabled)
2712 2713 2714
		return 0;

	/* HSW allows LP1+ watermarks even with multiple pipes */
2715
	if (level == 0 || config->num_pipes_active > 1) {
2716
		fifo_size /= INTEL_NUM_PIPES(dev_priv);
2717 2718 2719 2720 2721 2722

		/*
		 * For some reason the non self refresh
		 * FIFO size is only half of the self
		 * refresh FIFO size on ILK/SNB.
		 */
2723
		if (DISPLAY_VER(dev_priv) <= 6)
2724 2725 2726
			fifo_size /= 2;
	}

2727
	if (config->sprites_enabled) {
2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738
		/* level 0 is always calculated with 1:1 split */
		if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
			if (is_sprite)
				fifo_size *= 5;
			fifo_size /= 6;
		} else {
			fifo_size /= 2;
		}
	}

	/* clamp to max that the registers can hold */
2739
	return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2740 2741 2742
}

/* Calculate the maximum cursor plane watermark */
2743
static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2744 2745
				      int level,
				      const struct intel_wm_config *config)
2746 2747
{
	/* HSW LP1+ watermarks w/ multiple pipes */
2748
	if (level > 0 && config->num_pipes_active > 1)
2749 2750 2751
		return 64;

	/* otherwise just report max that registers can hold */
2752
	return ilk_cursor_wm_reg_max(dev_priv, level);
2753 2754
}

2755
static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2756 2757 2758
				    int level,
				    const struct intel_wm_config *config,
				    enum intel_ddb_partitioning ddb_partitioning,
2759
				    struct ilk_wm_maximums *max)
2760
{
2761 2762 2763 2764
	max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
	max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
	max->cur = ilk_cursor_wm_max(dev_priv, level, config);
	max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2765 2766
}

2767
static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2768 2769 2770
					int level,
					struct ilk_wm_maximums *max)
{
2771 2772 2773 2774
	max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
	max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
	max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
	max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2775 2776
}

2777
static bool ilk_validate_wm_level(int level,
2778
				  const struct ilk_wm_maximums *max,
2779
				  struct intel_wm_level *result)
2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808
{
	bool ret;

	/* already determined to be invalid? */
	if (!result->enable)
		return false;

	result->enable = result->pri_val <= max->pri &&
			 result->spr_val <= max->spr &&
			 result->cur_val <= max->cur;

	ret = result->enable;

	/*
	 * HACK until we can pre-compute everything,
	 * and thus fail gracefully if LP0 watermarks
	 * are exceeded...
	 */
	if (level == 0 && !result->enable) {
		if (result->pri_val > max->pri)
			DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
				      level, result->pri_val, max->pri);
		if (result->spr_val > max->spr)
			DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
				      level, result->spr_val, max->spr);
		if (result->cur_val > max->cur)
			DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
				      level, result->cur_val, max->cur);

2809 2810 2811
		result->pri_val = min_t(u32, result->pri_val, max->pri);
		result->spr_val = min_t(u32, result->spr_val, max->spr);
		result->cur_val = min_t(u32, result->cur_val, max->cur);
2812 2813 2814 2815 2816 2817
		result->enable = true;
	}

	return ret;
}

2818
static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2819
				 const struct intel_crtc *crtc,
2820
				 int level,
2821
				 struct intel_crtc_state *crtc_state,
2822 2823 2824
				 const struct intel_plane_state *pristate,
				 const struct intel_plane_state *sprstate,
				 const struct intel_plane_state *curstate,
2825
				 struct intel_wm_level *result)
2826
{
2827 2828 2829
	u16 pri_latency = dev_priv->wm.pri_latency[level];
	u16 spr_latency = dev_priv->wm.spr_latency[level];
	u16 cur_latency = dev_priv->wm.cur_latency[level];
2830 2831 2832 2833 2834 2835 2836 2837

	/* WM1+ latency values stored in 0.5us units */
	if (level > 0) {
		pri_latency *= 5;
		spr_latency *= 5;
		cur_latency *= 5;
	}

2838
	if (pristate) {
2839
		result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2840
						     pri_latency, level);
2841
		result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2842 2843 2844
	}

	if (sprstate)
2845
		result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2846 2847

	if (curstate)
2848
		result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2849

2850 2851 2852
	result->enable = true;
}

2853
static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2854
				  u16 wm[8])
2855
{
2856 2857
	struct intel_uncore *uncore = &dev_priv->uncore;

2858
	if (DISPLAY_VER(dev_priv) >= 9) {
2859
		u32 val;
2860
		int ret, i;
2861
		int level, max_level = ilk_wm_max_level(dev_priv);
2862 2863 2864 2865 2866

		/* read the first set of memory latencies[0:3] */
		val = 0; /* data0 to be programmed to 0 for first set */
		ret = sandybridge_pcode_read(dev_priv,
					     GEN9_PCODE_READ_MEM_LATENCY,
2867
					     &val, NULL);
2868 2869

		if (ret) {
2870 2871
			drm_err(&dev_priv->drm,
				"SKL Mailbox read error = %d\n", ret);
2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886
			return;
		}

		wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
		wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
				GEN9_MEM_LATENCY_LEVEL_MASK;
		wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
				GEN9_MEM_LATENCY_LEVEL_MASK;
		wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
				GEN9_MEM_LATENCY_LEVEL_MASK;

		/* read the second set of memory latencies[4:7] */
		val = 1; /* data0 to be programmed to 1 for second set */
		ret = sandybridge_pcode_read(dev_priv,
					     GEN9_PCODE_READ_MEM_LATENCY,
2887
					     &val, NULL);
2888
		if (ret) {
2889 2890
			drm_err(&dev_priv->drm,
				"SKL Mailbox read error = %d\n", ret);
2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901
			return;
		}

		wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
		wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
				GEN9_MEM_LATENCY_LEVEL_MASK;
		wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
				GEN9_MEM_LATENCY_LEVEL_MASK;
		wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
				GEN9_MEM_LATENCY_LEVEL_MASK;

2902 2903 2904 2905 2906 2907 2908 2909 2910
		/*
		 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
		 * need to be disabled. We make sure to sanitize the values out
		 * of the punit to satisfy this requirement.
		 */
		for (level = 1; level <= max_level; level++) {
			if (wm[level] == 0) {
				for (i = level + 1; i <= max_level; i++)
					wm[i] = 0;
2911 2912 2913

				max_level = level - 1;

2914 2915 2916 2917
				break;
			}
		}

2918
		/*
2919
		 * WaWmMemoryReadLatency
2920
		 *
2921
		 * punit doesn't take into account the read latency so we need
2922 2923
		 * to add proper adjustement to each valid level we retrieve
		 * from the punit when level 0 response data is 0us.
2924
		 */
2925
		if (wm[0] == 0) {
2926 2927
			u8 adjust = DISPLAY_VER(dev_priv) >= 12 ? 3 : 2;

2928
			for (level = 0; level <= max_level; level++)
2929
				wm[level] += adjust;
2930 2931
		}

2932 2933 2934 2935 2936 2937
		/*
		 * WA Level-0 adjustment for 16GB DIMMs: SKL+
		 * If we could not get dimm info enable this WA to prevent from
		 * any underrun. If not able to get Dimm info assume 16GB dimm
		 * to avoid any underrun.
		 */
2938
		if (dev_priv->dram_info.wm_lv_0_adjust_needed)
2939
			wm[0] += 1;
2940
	} else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2941
		u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2942 2943 2944 2945

		wm[0] = (sskpd >> 56) & 0xFF;
		if (wm[0] == 0)
			wm[0] = sskpd & 0xF;
2946 2947 2948 2949
		wm[1] = (sskpd >> 4) & 0xFF;
		wm[2] = (sskpd >> 12) & 0xFF;
		wm[3] = (sskpd >> 20) & 0x1FF;
		wm[4] = (sskpd >> 32) & 0x1FF;
2950
	} else if (DISPLAY_VER(dev_priv) >= 6) {
2951
		u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2952 2953 2954 2955 2956

		wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
		wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
		wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
		wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2957
	} else if (DISPLAY_VER(dev_priv) >= 5) {
2958
		u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2959 2960 2961 2962 2963

		/* ILK primary LP0 latency is 700 ns */
		wm[0] = 7;
		wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
		wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2964 2965
	} else {
		MISSING_CASE(INTEL_DEVID(dev_priv));
2966 2967 2968
	}
}

2969
static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2970
				       u16 wm[5])
2971 2972
{
	/* ILK sprite LP0 latency is 1300 ns */
2973
	if (DISPLAY_VER(dev_priv) == 5)
2974 2975 2976
		wm[0] = 13;
}

2977
static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2978
				       u16 wm[5])
2979 2980
{
	/* ILK cursor LP0 latency is 1300 ns */
2981
	if (DISPLAY_VER(dev_priv) == 5)
2982 2983 2984
		wm[0] = 13;
}

2985
int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2986 2987
{
	/* how many WM levels are we expecting */
2988 2989 2990
	if (HAS_HW_SAGV_WM(dev_priv))
		return 5;
	else if (DISPLAY_VER(dev_priv) >= 9)
2991
		return 7;
2992
	else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2993
		return 4;
2994
	else if (DISPLAY_VER(dev_priv) >= 6)
2995
		return 3;
2996
	else
2997 2998
		return 2;
}
2999

3000
static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
3001
				   const char *name,
Linus Torvalds's avatar
Linus Torvalds committed
3002
				   const u16 wm[])
3003
{
3004
	int level, max_level = ilk_wm_max_level(dev_priv);
3005 3006 3007 3008 3009

	for (level = 0; level <= max_level; level++) {
		unsigned int latency = wm[level];

		if (latency == 0) {
3010 3011 3012
			drm_dbg_kms(&dev_priv->drm,
				    "%s WM%d latency not provided\n",
				    name, level);
3013 3014 3015
			continue;
		}

3016 3017 3018 3019
		/*
		 * - latencies are in us on gen9.
		 * - before then, WM1+ latency values are in 0.5us units
		 */
3020
		if (DISPLAY_VER(dev_priv) >= 9)
3021 3022
			latency *= 10;
		else if (level > 0)
3023 3024
			latency *= 5;

3025 3026 3027
		drm_dbg_kms(&dev_priv->drm,
			    "%s WM%d latency %u (%u.%u usec)\n", name, level,
			    wm[level], latency / 10, latency % 10);
3028 3029 3030
	}
}

3031
static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3032
				    u16 wm[5], u16 min)
3033
{
3034
	int level, max_level = ilk_wm_max_level(dev_priv);
3035 3036 3037 3038 3039 3040

	if (wm[0] >= min)
		return false;

	wm[0] = max(wm[0], min);
	for (level = 1; level <= max_level; level++)
3041
		wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3042 3043 3044 3045

	return true;
}

3046
static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060
{
	bool changed;

	/*
	 * The BIOS provided WM memory latency values are often
	 * inadequate for high resolution displays. Adjust them.
	 */
	changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
		ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
		ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);

	if (!changed)
		return;

3061 3062
	drm_dbg_kms(&dev_priv->drm,
		    "WM latency values increased to avoid potential underruns\n");
3063 3064 3065
	intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3066 3067
}

3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089
static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
{
	/*
	 * On some SNB machines (Thinkpad X220 Tablet at least)
	 * LP3 usage can cause vblank interrupts to be lost.
	 * The DEIIR bit will go high but it looks like the CPU
	 * never gets interrupted.
	 *
	 * It's not clear whether other interrupt source could
	 * be affected or if this is somehow limited to vblank
	 * interrupts only. To play it safe we disable LP3
	 * watermarks entirely.
	 */
	if (dev_priv->wm.pri_latency[3] == 0 &&
	    dev_priv->wm.spr_latency[3] == 0 &&
	    dev_priv->wm.cur_latency[3] == 0)
		return;

	dev_priv->wm.pri_latency[3] = 0;
	dev_priv->wm.spr_latency[3] = 0;
	dev_priv->wm.cur_latency[3] = 0;

3090 3091
	drm_dbg_kms(&dev_priv->drm,
		    "LP3 watermarks disabled due to potential for lost interrupts\n");
3092 3093 3094 3095 3096
	intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
}

3097
static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3098
{
3099
	intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3100 3101 3102 3103 3104 3105

	memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
	       sizeof(dev_priv->wm.pri_latency));
	memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
	       sizeof(dev_priv->wm.pri_latency));

3106
	intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3107
	intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3108

3109 3110 3111
	intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3112

3113
	if (DISPLAY_VER(dev_priv) == 6) {
3114
		snb_wm_latency_quirk(dev_priv);
3115 3116
		snb_wm_lp3_irq_quirk(dev_priv);
	}
3117 3118
}

3119
static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3120
{
3121
	intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3122
	intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3123 3124
}

3125
static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136
				 struct intel_pipe_wm *pipe_wm)
{
	/* LP0 watermark maximums depend on this pipe alone */
	const struct intel_wm_config config = {
		.num_pipes_active = 1,
		.sprites_enabled = pipe_wm->sprites_enabled,
		.sprites_scaled = pipe_wm->sprites_scaled,
	};
	struct ilk_wm_maximums max;

	/* LP0 watermarks always use 1/2 DDB partitioning */
3137
	ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3138 3139 3140

	/* At least LP0 must be valid */
	if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3141
		drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3142 3143 3144 3145 3146 3147
		return false;
	}

	return true;
}

3148
/* Compute new watermarks for the pipe */
3149 3150
static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
			       struct intel_crtc *crtc)
3151
{
3152 3153 3154
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
	struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
3155
	struct intel_pipe_wm *pipe_wm;
3156 3157
	struct intel_plane *plane;
	const struct intel_plane_state *plane_state;
3158 3159 3160
	const struct intel_plane_state *pristate = NULL;
	const struct intel_plane_state *sprstate = NULL;
	const struct intel_plane_state *curstate = NULL;
3161
	int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3162
	struct ilk_wm_maximums max;
3163

3164
	pipe_wm = &crtc_state->wm.ilk.optimal;
3165

3166 3167 3168 3169 3170 3171 3172
	intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
		if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
			pristate = plane_state;
		else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
			sprstate = plane_state;
		else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
			curstate = plane_state;
3173 3174
	}

3175
	pipe_wm->pipe_enabled = crtc_state->hw.active;
3176
	if (sprstate) {
3177 3178 3179 3180
		pipe_wm->sprites_enabled = sprstate->uapi.visible;
		pipe_wm->sprites_scaled = sprstate->uapi.visible &&
			(drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
			 drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
3181 3182
	}

3183 3184
	usable_level = max_level;

3185
	/* ILK/SNB: LP2+ watermarks only w/o sprites */
3186
	if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3187
		usable_level = 1;
3188 3189

	/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3190
	if (pipe_wm->sprites_scaled)
3191
		usable_level = 0;
3192

3193
	memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3194
	ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3195
			     pristate, sprstate, curstate, &pipe_wm->wm[0]);
3196

3197
	if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3198
		return -EINVAL;
3199

3200
	ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3201

3202 3203
	for (level = 1; level <= usable_level; level++) {
		struct intel_wm_level *wm = &pipe_wm->wm[level];
3204

3205
		ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3206
				     pristate, sprstate, curstate, wm);
3207 3208 3209 3210 3211 3212

		/*
		 * Disable any watermark level that exceeds the
		 * register maximums since such watermarks are
		 * always invalid.
		 */
3213 3214 3215 3216
		if (!ilk_validate_wm_level(level, &max, wm)) {
			memset(wm, 0, sizeof(*wm));
			break;
		}
3217 3218
	}

3219
	return 0;
3220 3221
}

3222 3223 3224 3225 3226
/*
 * Build a set of 'intermediate' watermark values that satisfy both the old
 * state and the new state.  These can be programmed to the hardware
 * immediately.
 */
3227 3228 3229 3230 3231 3232 3233 3234 3235 3236
static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
				       struct intel_crtc *crtc)
{
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	struct intel_crtc_state *new_crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
	const struct intel_crtc_state *old_crtc_state =
		intel_atomic_get_old_crtc_state(state, crtc);
	struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
	const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
3237
	int level, max_level = ilk_wm_max_level(dev_priv);
3238 3239 3240 3241 3242 3243

	/*
	 * Start with the final, target watermarks, then combine with the
	 * currently active watermarks to get values that are safe both before
	 * and after the vblank.
	 */
3244 3245 3246 3247
	*a = new_crtc_state->wm.ilk.optimal;
	if (!new_crtc_state->hw.active ||
	    drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) ||
	    state->skip_intermediate_wm)
3248 3249
		return 0;

3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270
	a->pipe_enabled |= b->pipe_enabled;
	a->sprites_enabled |= b->sprites_enabled;
	a->sprites_scaled |= b->sprites_scaled;

	for (level = 0; level <= max_level; level++) {
		struct intel_wm_level *a_wm = &a->wm[level];
		const struct intel_wm_level *b_wm = &b->wm[level];

		a_wm->enable &= b_wm->enable;
		a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
		a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
		a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
		a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
	}

	/*
	 * We need to make sure that these merged watermark values are
	 * actually a valid configuration themselves.  If they're not,
	 * there's no safe way to transition from the old state to
	 * the new state, so we need to fail the atomic transaction.
	 */
3271
	if (!ilk_validate_pipe_wm(dev_priv, a))
3272 3273 3274 3275 3276 3277
		return -EINVAL;

	/*
	 * If our intermediate WM are identical to the final WM, then we can
	 * omit the post-vblank programming; only update if it's different.
	 */
3278 3279
	if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
		new_crtc_state->wm.need_postvbl_update = true;
3280 3281 3282 3283

	return 0;
}

3284 3285 3286
/*
 * Merge the watermarks from all active pipes for a specific level.
 */
3287
static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3288 3289 3290
			       int level,
			       struct intel_wm_level *ret_wm)
{
3291
	const struct intel_crtc *crtc;
3292

3293 3294
	ret_wm->enable = true;

3295 3296
	for_each_intel_crtc(&dev_priv->drm, crtc) {
		const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3297 3298 3299 3300
		const struct intel_wm_level *wm = &active->wm[level];

		if (!active->pipe_enabled)
			continue;
3301

3302 3303 3304 3305 3306
		/*
		 * The watermark values may have been used in the past,
		 * so we must maintain them in the registers for some
		 * time even if the level is now disabled.
		 */
3307
		if (!wm->enable)
3308
			ret_wm->enable = false;
3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319

		ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
		ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
		ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
		ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
	}
}

/*
 * Merge all low power watermarks for all active pipes.
 */
3320
static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3321
			 const struct intel_wm_config *config,
3322
			 const struct ilk_wm_maximums *max,
3323 3324
			 struct intel_pipe_wm *merged)
{
3325
	int level, max_level = ilk_wm_max_level(dev_priv);
3326
	int last_enabled_level = max_level;
3327

3328
	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3329
	if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3330
	    config->num_pipes_active > 1)
3331
		last_enabled_level = 0;
3332

3333
	/* ILK: FBC WM must be disabled always */
3334
	merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3335 3336 3337 3338 3339

	/* merge each WM1+ level */
	for (level = 1; level <= max_level; level++) {
		struct intel_wm_level *wm = &merged->wm[level];

3340
		ilk_merge_wm_level(dev_priv, level, wm);
3341

3342 3343 3344 3345 3346
		if (level > last_enabled_level)
			wm->enable = false;
		else if (!ilk_validate_wm_level(level, max, wm))
			/* make sure all following levels get disabled */
			last_enabled_level = level - 1;
3347 3348 3349 3350 3351 3352

		/*
		 * The spec says it is preferred to disable
		 * FBC WMs instead of disabling a WM level.
		 */
		if (wm->fbc_val > max->fbc) {
3353 3354
			if (wm->enable)
				merged->fbc_wm_enabled = false;
3355 3356 3357
			wm->fbc_val = 0;
		}
	}
3358 3359 3360 3361 3362 3363 3364

	/* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
	/*
	 * FIXME this is racy. FBC might get enabled later.
	 * What we should check here is whether FBC can be
	 * enabled sometime later.
	 */
3365
	if (DISPLAY_VER(dev_priv) == 5 && !merged->fbc_wm_enabled &&
3366
	    intel_fbc_is_active(dev_priv)) {
3367 3368 3369 3370 3371 3372
		for (level = 2; level <= max_level; level++) {
			struct intel_wm_level *wm = &merged->wm[level];

			wm->enable = false;
		}
	}
3373 3374
}

3375 3376 3377 3378 3379 3380
static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
{
	/* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
	return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
}

3381
/* The value we need to program into the WM_LPx latency field */
3382 3383
static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
				      int level)
3384
{
3385
	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3386 3387 3388 3389 3390
		return 2 * level;
	else
		return dev_priv->wm.pri_latency[level];
}

3391
static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3392
				   const struct intel_pipe_wm *merged,
3393
				   enum intel_ddb_partitioning partitioning,
3394
				   struct ilk_wm_values *results)
3395
{
3396
	struct intel_crtc *crtc;
3397
	int level, wm_lp;
3398

3399
	results->enable_fbc_wm = merged->fbc_wm_enabled;
3400
	results->partitioning = partitioning;
3401

3402
	/* LP1+ register values */
3403
	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3404
		const struct intel_wm_level *r;
3405

3406
		level = ilk_wm_lp_to_level(wm_lp, merged);
3407

3408
		r = &merged->wm[level];
3409

3410 3411 3412 3413 3414
		/*
		 * Maintain the watermark values even if the level is
		 * disabled. Doing otherwise could cause underruns.
		 */
		results->wm_lp[wm_lp - 1] =
3415
			(ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3416 3417 3418
			(r->pri_val << WM1_LP_SR_SHIFT) |
			r->cur_val;

3419 3420 3421
		if (r->enable)
			results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;

3422
		if (DISPLAY_VER(dev_priv) >= 8)
3423 3424 3425 3426 3427 3428
			results->wm_lp[wm_lp - 1] |=
				r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
		else
			results->wm_lp[wm_lp - 1] |=
				r->fbc_val << WM1_LP_FBC_SHIFT;

3429 3430 3431 3432
		/*
		 * Always set WM1S_LP_EN when spr_val != 0, even if the
		 * level is disabled. Doing otherwise could cause underruns.
		 */
3433
		if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3434
			drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3435 3436 3437
			results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
		} else
			results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3438
	}
3439

3440
	/* LP0 register values */
3441 3442 3443
	for_each_intel_crtc(&dev_priv->drm, crtc) {
		enum pipe pipe = crtc->pipe;
		const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3444
		const struct intel_wm_level *r = &pipe_wm->wm[0];
3445

3446
		if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3447
			continue;
3448

3449 3450 3451 3452
		results->wm_pipe[pipe] =
			(r->pri_val << WM0_PIPE_PLANE_SHIFT) |
			(r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
			r->cur_val;
3453 3454 3455
	}
}

3456 3457
/* Find the result with the highest level enabled. Check for enable_fbc_wm in
 * case both are at the same level. Prefer r1 in case they're the same. */
3458 3459 3460 3461
static struct intel_pipe_wm *
ilk_find_best_result(struct drm_i915_private *dev_priv,
		     struct intel_pipe_wm *r1,
		     struct intel_pipe_wm *r2)
3462
{
3463
	int level, max_level = ilk_wm_max_level(dev_priv);
3464
	int level1 = 0, level2 = 0;
3465

3466 3467 3468 3469 3470
	for (level = 1; level <= max_level; level++) {
		if (r1->wm[level].enable)
			level1 = level;
		if (r2->wm[level].enable)
			level2 = level;
3471 3472
	}

3473 3474
	if (level1 == level2) {
		if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3475 3476 3477
			return r2;
		else
			return r1;
3478
	} else if (level1 > level2) {
3479 3480 3481 3482 3483 3484
		return r1;
	} else {
		return r2;
	}
}

3485 3486 3487 3488 3489 3490 3491
/* dirty bits used to track which watermarks need changes */
#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
#define WM_DIRTY_FBC (1 << 24)
#define WM_DIRTY_DDB (1 << 25)

3492
static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3493 3494
					 const struct ilk_wm_values *old,
					 const struct ilk_wm_values *new)
3495 3496 3497 3498 3499
{
	unsigned int dirty = 0;
	enum pipe pipe;
	int wm_lp;

3500
	for_each_pipe(dev_priv, pipe) {
3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537
		if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
			dirty |= WM_DIRTY_PIPE(pipe);
			/* Must disable LP1+ watermarks too */
			dirty |= WM_DIRTY_LP_ALL;
		}
	}

	if (old->enable_fbc_wm != new->enable_fbc_wm) {
		dirty |= WM_DIRTY_FBC;
		/* Must disable LP1+ watermarks too */
		dirty |= WM_DIRTY_LP_ALL;
	}

	if (old->partitioning != new->partitioning) {
		dirty |= WM_DIRTY_DDB;
		/* Must disable LP1+ watermarks too */
		dirty |= WM_DIRTY_LP_ALL;
	}

	/* LP1+ watermarks already deemed dirty, no need to continue */
	if (dirty & WM_DIRTY_LP_ALL)
		return dirty;

	/* Find the lowest numbered LP1+ watermark in need of an update... */
	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
		    old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
			break;
	}

	/* ...and mark it and all higher numbered LP1+ watermarks as dirty */
	for (; wm_lp <= 3; wm_lp++)
		dirty |= WM_DIRTY_LP(wm_lp);

	return dirty;
}

3538 3539
static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
			       unsigned int dirty)
3540
{
3541
	struct ilk_wm_values *previous = &dev_priv->wm.hw;
3542
	bool changed = false;
3543

3544 3545
	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
		previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3546
		intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3547
		changed = true;
3548 3549 3550
	}
	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
		previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3551
		intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3552
		changed = true;
3553 3554 3555
	}
	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
		previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3556
		intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3557
		changed = true;
3558
	}
3559

3560 3561 3562 3563
	/*
	 * Don't touch WM1S_LP_EN here.
	 * Doing so could cause underruns.
	 */
3564

3565 3566 3567 3568 3569 3570 3571
	return changed;
}

/*
 * The spec says we shouldn't write when we don't need, because every write
 * causes WMs to be re-evaluated, expending some power.
 */
3572 3573
static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
				struct ilk_wm_values *results)
3574
{
3575
	struct ilk_wm_values *previous = &dev_priv->wm.hw;
3576
	unsigned int dirty;
3577
	u32 val;
3578

3579
	dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3580 3581 3582 3583 3584
	if (!dirty)
		return;

	_ilk_disable_lp_wm(dev_priv, dirty);

3585
	if (dirty & WM_DIRTY_PIPE(PIPE_A))
3586
		intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3587
	if (dirty & WM_DIRTY_PIPE(PIPE_B))
3588
		intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3589
	if (dirty & WM_DIRTY_PIPE(PIPE_C))
3590
		intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3591

3592
	if (dirty & WM_DIRTY_DDB) {
3593
		if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3594
			val = intel_uncore_read(&dev_priv->uncore, WM_MISC);
3595 3596 3597 3598
			if (results->partitioning == INTEL_DDB_PART_1_2)
				val &= ~WM_MISC_DATA_PARTITION_5_6;
			else
				val |= WM_MISC_DATA_PARTITION_5_6;
3599
			intel_uncore_write(&dev_priv->uncore, WM_MISC, val);
3600
		} else {
3601
			val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
3602 3603 3604 3605
			if (results->partitioning == INTEL_DDB_PART_1_2)
				val &= ~DISP_DATA_PARTITION_5_6;
			else
				val |= DISP_DATA_PARTITION_5_6;
3606
			intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
3607
		}
3608 3609
	}

3610
	if (dirty & WM_DIRTY_FBC) {
3611
		val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL);
3612 3613 3614 3615
		if (results->enable_fbc_wm)
			val &= ~DISP_FBC_WM_DIS;
		else
			val |= DISP_FBC_WM_DIS;
3616
		intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, val);
3617 3618
	}

3619 3620
	if (dirty & WM_DIRTY_LP(1) &&
	    previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3621
		intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3622

3623
	if (DISPLAY_VER(dev_priv) >= 7) {
3624
		if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3625
			intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3626
		if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3627
			intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3628
	}
3629

3630
	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3631
		intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3632
	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3633
		intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3634
	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3635
		intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3636 3637

	dev_priv->wm.hw = *results;
3638 3639
}

3640
bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3641 3642 3643 3644
{
	return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
}

3645
u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3646
{
3647 3648
	u8 enabled_slices = 0;
	enum dbuf_slice slice;
3649

3650 3651 3652 3653
	for_each_dbuf_slice(dev_priv, slice) {
		if (intel_uncore_read(&dev_priv->uncore,
				      DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
			enabled_slices |= BIT(slice);
3654
	}
3655

3656
	return enabled_slices;
3657 3658
}

3659 3660 3661 3662
/*
 * FIXME: We still don't have the proper code detect if we need to apply the WA,
 * so assume we'll always need it in order to avoid underruns.
 */
3663
static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3664
{
3665
	return DISPLAY_VER(dev_priv) == 9;
3666 3667
}

3668 3669 3670
static bool
intel_has_sagv(struct drm_i915_private *dev_priv)
{
3671
	return DISPLAY_VER(dev_priv) >= 9 && !IS_LP(dev_priv) &&
3672
		dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3673 3674
}

3675 3676 3677
static void
skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
{
3678
	if (DISPLAY_VER(dev_priv) >= 12) {
3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689
		u32 val = 0;
		int ret;

		ret = sandybridge_pcode_read(dev_priv,
					     GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
					     &val, NULL);
		if (!ret) {
			dev_priv->sagv_block_time_us = val;
			return;
		}

3690
		drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3691
	} else if (DISPLAY_VER(dev_priv) == 11) {
3692 3693
		dev_priv->sagv_block_time_us = 10;
		return;
3694
	} else if (DISPLAY_VER(dev_priv) == 10) {
3695 3696
		dev_priv->sagv_block_time_us = 20;
		return;
3697
	} else if (DISPLAY_VER(dev_priv) == 9) {
3698 3699 3700
		dev_priv->sagv_block_time_us = 30;
		return;
	} else {
3701
		MISSING_CASE(DISPLAY_VER(dev_priv));
3702 3703 3704 3705 3706 3707
	}

	/* Default to an unusable block time */
	dev_priv->sagv_block_time_us = -1;
}

3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718
/*
 * SAGV dynamically adjusts the system agent voltage and clock frequencies
 * depending on power and performance requirements. The display engine access
 * to system memory is blocked during the adjustment time. Because of the
 * blocking time, having this enabled can cause full system hangs and/or pipe
 * underruns if we don't meet all of the following requirements:
 *
 *  - <= 1 pipe enabled
 *  - All planes can enable watermarks for latencies >= SAGV engine block time
 *  - We're not using an interlaced display configuration
 */
3719
static int
3720
intel_enable_sagv(struct drm_i915_private *dev_priv)
3721 3722 3723
{
	int ret;

3724 3725 3726 3727
	if (!intel_has_sagv(dev_priv))
		return 0;

	if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3728 3729
		return 0;

3730
	drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3731 3732 3733
	ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
				      GEN9_SAGV_ENABLE);

3734
	/* We don't need to wait for SAGV when enabling */
3735 3736 3737

	/*
	 * Some skl systems, pre-release machines in particular,
3738
	 * don't actually have SAGV.
3739
	 */
3740
	if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3741
		drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3742
		dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3743 3744
		return 0;
	} else if (ret < 0) {
3745
		drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3746 3747 3748
		return ret;
	}

3749
	dev_priv->sagv_status = I915_SAGV_ENABLED;
3750 3751 3752
	return 0;
}

3753
static int
3754
intel_disable_sagv(struct drm_i915_private *dev_priv)
3755
{
3756
	int ret;
3757

3758 3759 3760 3761
	if (!intel_has_sagv(dev_priv))
		return 0;

	if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3762 3763
		return 0;

3764
	drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3765
	/* bspec says to keep retrying for at least 1 ms */
3766 3767 3768 3769
	ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
				GEN9_SAGV_DISABLE,
				GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
				1);
3770 3771
	/*
	 * Some skl systems, pre-release machines in particular,
3772
	 * don't actually have SAGV.
3773
	 */
3774
	if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3775
		drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3776
		dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3777
		return 0;
3778
	} else if (ret < 0) {
3779
		drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3780
		return ret;
3781 3782
	}

3783
	dev_priv->sagv_status = I915_SAGV_DISABLED;
3784 3785 3786
	return 0;
}

3787 3788 3789
void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
{
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3790
	const struct intel_bw_state *new_bw_state;
3791 3792
	const struct intel_bw_state *old_bw_state;
	u32 new_mask = 0;
3793

3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807
	/*
	 * Just return if we can't control SAGV or don't have it.
	 * This is different from situation when we have SAGV but just can't
	 * afford it due to DBuf limitation - in case if SAGV is completely
	 * disabled in a BIOS, we are not even allowed to send a PCode request,
	 * as it will throw an error. So have to check it here.
	 */
	if (!intel_has_sagv(dev_priv))
		return;

	new_bw_state = intel_atomic_get_new_bw_state(state);
	if (!new_bw_state)
		return;

3808
	if (DISPLAY_VER(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) {
3809
		intel_disable_sagv(dev_priv);
3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835
		return;
	}

	old_bw_state = intel_atomic_get_old_bw_state(state);
	/*
	 * Nothing to mask
	 */
	if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
		return;

	new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;

	/*
	 * If new mask is zero - means there is nothing to mask,
	 * we can only unmask, which should be done in unmask.
	 */
	if (!new_mask)
		return;

	/*
	 * Restrict required qgv points before updating the configuration.
	 * According to BSpec we can't mask and unmask qgv points at the same
	 * time. Also masking should be done before updating the configuration
	 * and unmasking afterwards.
	 */
	icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3836 3837 3838 3839 3840
}

void intel_sagv_post_plane_update(struct intel_atomic_state *state)
{
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3841
	const struct intel_bw_state *new_bw_state;
3842 3843
	const struct intel_bw_state *old_bw_state;
	u32 new_mask = 0;
3844 3845 3846 3847 3848 3849 3850 3851 3852 3853

	/*
	 * Just return if we can't control SAGV or don't have it.
	 * This is different from situation when we have SAGV but just can't
	 * afford it due to DBuf limitation - in case if SAGV is completely
	 * disabled in a BIOS, we are not even allowed to send a PCode request,
	 * as it will throw an error. So have to check it here.
	 */
	if (!intel_has_sagv(dev_priv))
		return;
3854

3855 3856 3857 3858
	new_bw_state = intel_atomic_get_new_bw_state(state);
	if (!new_bw_state)
		return;

3859
	if (DISPLAY_VER(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) {
3860
		intel_enable_sagv(dev_priv);
3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879
		return;
	}

	old_bw_state = intel_atomic_get_old_bw_state(state);
	/*
	 * Nothing to unmask
	 */
	if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
		return;

	new_mask = new_bw_state->qgv_points_mask;

	/*
	 * Allow required qgv points after updating the configuration.
	 * According to BSpec we can't mask and unmask qgv points at the same
	 * time. Also masking should be done before updating the configuration
	 * and unmasking afterwards.
	 */
	icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3880 3881
}

3882
static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3883
{
3884
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3885
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3886
	enum plane_id plane_id;
3887
	int max_level = INT_MAX;
3888

3889 3890 3891
	if (!intel_has_sagv(dev_priv))
		return false;

3892
	if (!crtc_state->hw.active)
3893
		return true;
3894

3895
	if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
3896 3897
		return false;

3898
	for_each_plane_id_on_crtc(crtc, plane_id) {
3899
		const struct skl_plane_wm *wm =
3900 3901
			&crtc_state->wm.skl.optimal.planes[plane_id];
		int level;
3902

3903
		/* Skip this plane if it's not enabled */
3904
		if (!wm->wm[0].enable)
3905 3906 3907
			continue;

		/* Find the highest enabled wm level for this plane */
3908
		for (level = ilk_wm_max_level(dev_priv);
3909
		     !wm->wm[level].enable; --level)
3910 3911
		     { }

3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923
		/* Highest common enabled wm level for all planes */
		max_level = min(level, max_level);
	}

	/* No enabled planes? */
	if (max_level == INT_MAX)
		return true;

	for_each_plane_id_on_crtc(crtc, plane_id) {
		const struct skl_plane_wm *wm =
			&crtc_state->wm.skl.optimal.planes[plane_id];

3924
		/*
3925 3926
		 * All enabled planes must have enabled a common wm level that
		 * can tolerate memory latencies higher than sagv_block_time_us
3927
		 */
3928
		if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
3929 3930 3931 3932 3933 3934
			return false;
	}

	return true;
}

3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946
static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
{
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
	enum plane_id plane_id;

	if (!crtc_state->hw.active)
		return true;

	for_each_plane_id_on_crtc(crtc, plane_id) {
		const struct skl_plane_wm *wm =
			&crtc_state->wm.skl.optimal.planes[plane_id];

3947
		if (wm->wm[0].enable && !wm->sagv.wm0.enable)
3948 3949 3950 3951 3952 3953
			return false;
	}

	return true;
}

3954 3955
static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
{
3956 3957 3958
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);

3959
	if (DISPLAY_VER(dev_priv) >= 12)
3960 3961 3962
		return tgl_crtc_can_enable_sagv(crtc_state);
	else
		return skl_crtc_can_enable_sagv(crtc_state);
3963 3964
}

3965 3966
bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
			   const struct intel_bw_state *bw_state)
3967
{
3968
	if (DISPLAY_VER(dev_priv) < 11 &&
3969
	    bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
3970 3971
		return false;

3972 3973 3974 3975 3976
	return bw_state->pipe_sagv_reject == 0;
}

static int intel_compute_sagv_mask(struct intel_atomic_state *state)
{
3977
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3978
	int ret;
3979
	struct intel_crtc *crtc;
3980
	struct intel_crtc_state *new_crtc_state;
3981 3982 3983
	struct intel_bw_state *new_bw_state = NULL;
	const struct intel_bw_state *old_bw_state = NULL;
	int i;
3984

3985 3986 3987 3988 3989
	for_each_new_intel_crtc_in_state(state, crtc,
					 new_crtc_state, i) {
		new_bw_state = intel_atomic_get_bw_state(state);
		if (IS_ERR(new_bw_state))
			return PTR_ERR(new_bw_state);
3990

3991
		old_bw_state = intel_atomic_get_old_bw_state(state);
3992

3993 3994 3995 3996 3997
		if (intel_crtc_can_enable_sagv(new_crtc_state))
			new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
		else
			new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
	}
3998

3999 4000
	if (!new_bw_state)
		return 0;
4001

4002 4003
	new_bw_state->active_pipes =
		intel_calc_active_pipes(state, old_bw_state->active_pipes);
4004

4005 4006 4007 4008 4009 4010
	if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
		ret = intel_atomic_lock_global_state(&new_bw_state->base);
		if (ret)
			return ret;
	}

4011 4012 4013 4014 4015 4016 4017 4018 4019 4020
	for_each_new_intel_crtc_in_state(state, crtc,
					 new_crtc_state, i) {
		struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;

		/*
		 * We store use_sagv_wm in the crtc state rather than relying on
		 * that bw state since we have no convenient way to get at the
		 * latter from the plane commit hooks (especially in the legacy
		 * cursor case)
		 */
4021 4022 4023
		pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(dev_priv) &&
			DISPLAY_VER(dev_priv) >= 12 &&
			intel_can_enable_sagv(dev_priv, new_bw_state);
4024 4025
	}

4026 4027
	if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
	    intel_can_enable_sagv(dev_priv, old_bw_state)) {
4028 4029 4030 4031 4032 4033 4034 4035 4036 4037
		ret = intel_atomic_serialize_global_state(&new_bw_state->base);
		if (ret)
			return ret;
	} else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
		ret = intel_atomic_lock_global_state(&new_bw_state->base);
		if (ret)
			return ret;
	}

	return 0;
4038 4039
}

4040 4041
static int intel_dbuf_slice_size(struct drm_i915_private *dev_priv)
{
4042 4043
	return INTEL_INFO(dev_priv)->dbuf.size /
		hweight8(INTEL_INFO(dev_priv)->dbuf.slice_mask);
4044 4045
}

4046 4047 4048
static void
skl_ddb_entry_for_slices(struct drm_i915_private *dev_priv, u8 slice_mask,
			 struct skl_ddb_entry *ddb)
4049
{
4050
	int slice_size = intel_dbuf_slice_size(dev_priv);
4051

4052 4053 4054 4055 4056
	if (!slice_mask) {
		ddb->start = 0;
		ddb->end = 0;
		return;
	}
4057

4058 4059
	ddb->start = (ffs(slice_mask) - 1) * slice_size;
	ddb->end = fls(slice_mask) * slice_size;
4060

4061
	WARN_ON(ddb->start >= ddb->end);
4062
	WARN_ON(ddb->end > INTEL_INFO(dev_priv)->dbuf.size);
4063 4064
}

4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078
static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
{
	struct skl_ddb_entry ddb;

	if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
		slice_mask = BIT(DBUF_S1);
	else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
		slice_mask = BIT(DBUF_S3);

	skl_ddb_entry_for_slices(i915, slice_mask, &ddb);

	return ddb.start;
}

4079 4080 4081
u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
			    const struct skl_ddb_entry *entry)
{
4082 4083 4084
	int slice_size = intel_dbuf_slice_size(dev_priv);
	enum dbuf_slice start_slice, end_slice;
	u8 slice_mask = 0;
4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103

	if (!skl_ddb_entry_size(entry))
		return 0;

	start_slice = entry->start / slice_size;
	end_slice = (entry->end - 1) / slice_size;

	/*
	 * Per plane DDB entry can in a really worst case be on multiple slices
	 * but single entry is anyway contigious.
	 */
	while (start_slice <= end_slice) {
		slice_mask |= BIT(start_slice);
		start_slice++;
	}

	return slice_mask;
}

4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121
static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
{
	const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
	int hdisplay, vdisplay;

	if (!crtc_state->hw.active)
		return 0;

	/*
	 * Watermark/ddb requirement highly depends upon width of the
	 * framebuffer, So instead of allocating DDB equally among pipes
	 * distribute DDB based on resolution/width of the display.
	 */
	drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);

	return hdisplay;
}

4122 4123 4124 4125 4126 4127 4128 4129 4130
static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
				    enum pipe for_pipe,
				    unsigned int *weight_start,
				    unsigned int *weight_end,
				    unsigned int *weight_total)
{
	struct drm_i915_private *dev_priv =
		to_i915(dbuf_state->base.state->base.dev);
	enum pipe pipe;
4131

4132 4133 4134
	*weight_start = 0;
	*weight_end = 0;
	*weight_total = 0;
4135

4136 4137
	for_each_pipe(dev_priv, pipe) {
		int weight = dbuf_state->weight[pipe];
4138 4139 4140 4141 4142 4143 4144 4145

		/*
		 * Do not account pipes using other slice sets
		 * luckily as of current BSpec slice sets do not partially
		 * intersect(pipes share either same one slice or same slice set
		 * i.e no partial intersection), so it is enough to check for
		 * equality for now.
		 */
4146
		if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
4147 4148
			continue;

4149 4150 4151 4152 4153 4154 4155
		*weight_total += weight;
		if (pipe < for_pipe) {
			*weight_start += weight;
			*weight_end += weight;
		} else if (pipe == for_pipe) {
			*weight_end += weight;
		}
4156
	}
4157 4158 4159
}

static int
4160
skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
4161
{
4162 4163
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	unsigned int weight_total, weight_start, weight_end;
4164 4165 4166 4167
	const struct intel_dbuf_state *old_dbuf_state =
		intel_atomic_get_old_dbuf_state(state);
	struct intel_dbuf_state *new_dbuf_state =
		intel_atomic_get_new_dbuf_state(state);
4168
	struct intel_crtc_state *crtc_state;
4169
	struct skl_ddb_entry ddb_slices;
4170
	enum pipe pipe = crtc->pipe;
4171
	unsigned int mbus_offset = 0;
4172 4173 4174 4175 4176
	u32 ddb_range_size;
	u32 dbuf_slice_mask;
	u32 start, end;
	int ret;

4177 4178 4179 4180
	if (new_dbuf_state->weight[pipe] == 0) {
		new_dbuf_state->ddb[pipe].start = 0;
		new_dbuf_state->ddb[pipe].end = 0;
		goto out;
4181 4182
	}

4183
	dbuf_slice_mask = new_dbuf_state->slices[pipe];
4184 4185

	skl_ddb_entry_for_slices(dev_priv, dbuf_slice_mask, &ddb_slices);
4186
	mbus_offset = mbus_ddb_offset(dev_priv, dbuf_slice_mask);
4187 4188
	ddb_range_size = skl_ddb_entry_size(&ddb_slices);

4189 4190
	intel_crtc_dbuf_weights(new_dbuf_state, pipe,
				&weight_start, &weight_end, &weight_total);
4191 4192 4193

	start = ddb_range_size * weight_start / weight_total;
	end = ddb_range_size * weight_end / weight_total;
4194

4195 4196
	new_dbuf_state->ddb[pipe].start = ddb_slices.start - mbus_offset + start;
	new_dbuf_state->ddb[pipe].end = ddb_slices.start - mbus_offset + end;
4197
out:
4198 4199
	if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
	    skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210
				&new_dbuf_state->ddb[pipe]))
		return 0;

	ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
	if (ret)
		return ret;

	crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
	if (IS_ERR(crtc_state))
		return PTR_ERR(crtc_state);

4211 4212 4213 4214 4215 4216
	/*
	 * Used for checking overlaps, so we need absolute
	 * offsets instead of MBUS relative offsets.
	 */
	crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
	crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
4217

4218
	drm_dbg_kms(&dev_priv->drm,
4219
		    "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
4220
		    crtc->base.base.id, crtc->base.name,
4221 4222 4223 4224
		    old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
		    old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
		    new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
		    old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
4225 4226

	return 0;
4227 4228
}

4229 4230 4231 4232 4233
static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
				 int width, const struct drm_format_info *format,
				 u64 modifier, unsigned int rotation,
				 u32 plane_pixel_rate, struct skl_wm_params *wp,
				 int color_plane);
4234
static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4235
				 int level,
4236
				 unsigned int latency,
4237 4238 4239 4240 4241 4242 4243
				 const struct skl_wm_params *wp,
				 const struct skl_wm_level *result_prev,
				 struct skl_wm_level *result /* out */);

static unsigned int
skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
		      int num_active)
4244
{
4245
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4246 4247 4248 4249 4250 4251 4252 4253 4254 4255
	int level, max_level = ilk_wm_max_level(dev_priv);
	struct skl_wm_level wm = {};
	int ret, min_ddb_alloc = 0;
	struct skl_wm_params wp;

	ret = skl_compute_wm_params(crtc_state, 256,
				    drm_format_info(DRM_FORMAT_ARGB8888),
				    DRM_FORMAT_MOD_LINEAR,
				    DRM_MODE_ROTATE_0,
				    crtc_state->pixel_rate, &wp, 0);
4256
	drm_WARN_ON(&dev_priv->drm, ret);
4257 4258

	for (level = 0; level <= max_level; level++) {
4259 4260 4261
		unsigned int latency = dev_priv->wm.skl_latency[level];

		skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm);
4262 4263 4264 4265 4266
		if (wm.min_ddb_alloc == U16_MAX)
			break;

		min_ddb_alloc = wm.min_ddb_alloc;
	}
4267

4268
	return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4269 4270
}

4271 4272
static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
				       struct skl_ddb_entry *entry, u32 reg)
4273
{
4274 4275
	entry->start = reg & DDB_ENTRY_MASK;
	entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
4276

4277 4278
	if (entry->end)
		entry->end += 1;
4279 4280
}

4281 4282 4283 4284
static void
skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
			   const enum pipe pipe,
			   const enum plane_id plane_id,
4285 4286
			   struct skl_ddb_entry *ddb_y,
			   struct skl_ddb_entry *ddb_uv)
4287
{
4288 4289
	u32 val, val2;
	u32 fourcc = 0;
4290 4291 4292

	/* Cursor doesn't support NV12/planar, so no extra calculation needed */
	if (plane_id == PLANE_CURSOR) {
4293
		val = intel_uncore_read(&dev_priv->uncore, CUR_BUF_CFG(pipe));
4294
		skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4295 4296 4297
		return;
	}

4298
	val = intel_uncore_read(&dev_priv->uncore, PLANE_CTL(pipe, plane_id));
4299 4300

	/* No DDB allocated for disabled planes */
4301 4302 4303 4304
	if (val & PLANE_CTL_ENABLE)
		fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
					      val & PLANE_CTL_ORDER_RGBX,
					      val & PLANE_CTL_ALPHA_MASK);
4305

4306
	if (DISPLAY_VER(dev_priv) >= 11) {
4307
		val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4308 4309
		skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
	} else {
4310 4311
		val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
		val2 = intel_uncore_read(&dev_priv->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id));
4312

4313 4314
		if (fourcc &&
		    drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4315 4316 4317 4318
			swap(val, val2);

		skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
		skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4319 4320 4321
	}
}

4322 4323 4324
void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
			       struct skl_ddb_entry *ddb_y,
			       struct skl_ddb_entry *ddb_uv)
4325
{
4326 4327 4328
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	enum intel_display_power_domain power_domain;
	enum pipe pipe = crtc->pipe;
4329
	intel_wakeref_t wakeref;
4330
	enum plane_id plane_id;
4331

4332
	power_domain = POWER_DOMAIN_PIPE(pipe);
4333 4334
	wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
	if (!wakeref)
4335
		return;
4336

4337 4338 4339 4340 4341
	for_each_plane_id_on_crtc(crtc, plane_id)
		skl_ddb_get_hw_plane_state(dev_priv, pipe,
					   plane_id,
					   &ddb_y[plane_id],
					   &ddb_uv[plane_id]);
4342

4343
	intel_display_power_put(dev_priv, power_domain, wakeref);
4344
}
4345

4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361
/*
 * Determines the downscale amount of a plane for the purposes of watermark calculations.
 * The bspec defines downscale amount as:
 *
 * """
 * Horizontal down scale amount = maximum[1, Horizontal source size /
 *                                           Horizontal destination size]
 * Vertical down scale amount = maximum[1, Vertical source size /
 *                                         Vertical destination size]
 * Total down scale amount = Horizontal down scale amount *
 *                           Vertical down scale amount
 * """
 *
 * Return value is provided in 16.16 fixed point form to retain fractional part.
 * Caller should take care of dividing & rounding off the value.
 */
4362
static uint_fixed_16_16_t
4363 4364
skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
			   const struct intel_plane_state *plane_state)
4365
{
4366
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4367
	u32 src_w, src_h, dst_w, dst_h;
4368 4369
	uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
	uint_fixed_16_16_t downscale_h, downscale_w;
4370

4371 4372
	if (drm_WARN_ON(&dev_priv->drm,
			!intel_wm_plane_visible(crtc_state, plane_state)))
4373
		return u32_to_fixed16(0);
4374

4375 4376 4377 4378 4379 4380 4381
	/*
	 * Src coordinates are already rotated by 270 degrees for
	 * the 90/270 degree plane rotation cases (to match the
	 * GTT mapping), hence no need to account for rotation here.
	 *
	 * n.b., src is 16.16 fixed point, dst is whole integer.
	 */
4382 4383 4384 4385
	src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
	src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
	dst_w = drm_rect_width(&plane_state->uapi.dst);
	dst_h = drm_rect_height(&plane_state->uapi.dst);
4386

4387 4388 4389 4390
	fp_w_ratio = div_fixed16(src_w, dst_w);
	fp_h_ratio = div_fixed16(src_h, dst_h);
	downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
	downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4391

4392
	return mul_fixed16(downscale_w, downscale_h);
4393 4394
}

4395 4396 4397
struct dbuf_slice_conf_entry {
	u8 active_pipes;
	u8 dbuf_mask[I915_MAX_PIPES];
4398
	bool join_mbus;
4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410
};

/*
 * Table taken from Bspec 12716
 * Pipes do have some preferred DBuf slice affinity,
 * plus there are some hardcoded requirements on how
 * those should be distributed for multipipe scenarios.
 * For more DBuf slices algorithm can get even more messy
 * and less readable, so decided to use a table almost
 * as is from BSpec itself - that way it is at least easier
 * to compare, change and check.
 */
4411
static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4412 4413 4414 4415 4416
/* Autogenerated with igt/tools/intel_dbuf_map tool: */
{
	{
		.active_pipes = BIT(PIPE_A),
		.dbuf_mask = {
4417 4418
			[PIPE_A] = BIT(DBUF_S1),
		},
4419 4420 4421 4422
	},
	{
		.active_pipes = BIT(PIPE_B),
		.dbuf_mask = {
4423 4424
			[PIPE_B] = BIT(DBUF_S1),
		},
4425 4426 4427 4428 4429
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
4430 4431
			[PIPE_B] = BIT(DBUF_S2),
		},
4432 4433 4434 4435
	},
	{
		.active_pipes = BIT(PIPE_C),
		.dbuf_mask = {
4436 4437
			[PIPE_C] = BIT(DBUF_S2),
		},
4438 4439 4440 4441 4442
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
4443 4444
			[PIPE_C] = BIT(DBUF_S2),
		},
4445 4446 4447 4448 4449
	},
	{
		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_B] = BIT(DBUF_S1),
4450 4451
			[PIPE_C] = BIT(DBUF_S2),
		},
4452 4453 4454 4455 4456 4457
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
			[PIPE_B] = BIT(DBUF_S1),
4458 4459
			[PIPE_C] = BIT(DBUF_S2),
		},
4460
	},
4461
	{}
4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473
};

/*
 * Table taken from Bspec 49255
 * Pipes do have some preferred DBuf slice affinity,
 * plus there are some hardcoded requirements on how
 * those should be distributed for multipipe scenarios.
 * For more DBuf slices algorithm can get even more messy
 * and less readable, so decided to use a table almost
 * as is from BSpec itself - that way it is at least easier
 * to compare, change and check.
 */
4474
static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4475 4476 4477 4478 4479
/* Autogenerated with igt/tools/intel_dbuf_map tool: */
{
	{
		.active_pipes = BIT(PIPE_A),
		.dbuf_mask = {
4480 4481
			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
		},
4482 4483 4484 4485
	},
	{
		.active_pipes = BIT(PIPE_B),
		.dbuf_mask = {
4486 4487
			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
		},
4488 4489 4490 4491 4492
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S2),
4493 4494
			[PIPE_B] = BIT(DBUF_S1),
		},
4495 4496 4497 4498
	},
	{
		.active_pipes = BIT(PIPE_C),
		.dbuf_mask = {
4499 4500
			[PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
		},
4501 4502 4503 4504 4505
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
4506 4507
			[PIPE_C] = BIT(DBUF_S2),
		},
4508 4509 4510 4511 4512
	},
	{
		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_B] = BIT(DBUF_S1),
4513 4514
			[PIPE_C] = BIT(DBUF_S2),
		},
4515 4516 4517 4518 4519 4520
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
			[PIPE_B] = BIT(DBUF_S1),
4521 4522
			[PIPE_C] = BIT(DBUF_S2),
		},
4523 4524 4525 4526
	},
	{
		.active_pipes = BIT(PIPE_D),
		.dbuf_mask = {
4527 4528
			[PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
		},
4529 4530 4531 4532 4533
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
4534 4535
			[PIPE_D] = BIT(DBUF_S2),
		},
4536 4537 4538 4539 4540
	},
	{
		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_B] = BIT(DBUF_S1),
4541 4542
			[PIPE_D] = BIT(DBUF_S2),
		},
4543 4544 4545 4546 4547 4548
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
			[PIPE_B] = BIT(DBUF_S1),
4549 4550
			[PIPE_D] = BIT(DBUF_S2),
		},
4551 4552 4553 4554 4555
	},
	{
		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_C] = BIT(DBUF_S1),
4556 4557
			[PIPE_D] = BIT(DBUF_S2),
		},
4558 4559 4560 4561 4562 4563
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
			[PIPE_C] = BIT(DBUF_S2),
4564 4565
			[PIPE_D] = BIT(DBUF_S2),
		},
4566 4567 4568 4569 4570 4571
	},
	{
		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_B] = BIT(DBUF_S1),
			[PIPE_C] = BIT(DBUF_S2),
4572 4573
			[PIPE_D] = BIT(DBUF_S2),
		},
4574 4575 4576 4577 4578 4579 4580
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
			[PIPE_B] = BIT(DBUF_S1),
			[PIPE_C] = BIT(DBUF_S2),
4581 4582
			[PIPE_D] = BIT(DBUF_S2),
		},
4583
	},
4584
	{}
4585 4586
};

4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697
static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
	{
		.active_pipes = BIT(PIPE_A),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
		},
	},
	{
		.active_pipes = BIT(PIPE_B),
		.dbuf_mask = {
			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
		},
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
			[PIPE_B] = BIT(DBUF_S2),
		},
	},
	{
		.active_pipes = BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
		},
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
		},
	},
	{
		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
		},
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
			[PIPE_B] = BIT(DBUF_S2),
			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
		},
	},
	{
		.active_pipes = BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
		},
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
		},
	},
	{
		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
		},
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
			[PIPE_B] = BIT(DBUF_S2),
			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
		},
	},
	{
		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_C] = BIT(DBUF_S3),
			[PIPE_D] = BIT(DBUF_S4),
		},
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
			[PIPE_C] = BIT(DBUF_S3),
			[PIPE_D] = BIT(DBUF_S4),
		},
	},
	{
		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
			[PIPE_C] = BIT(DBUF_S3),
			[PIPE_D] = BIT(DBUF_S4),
		},
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1),
			[PIPE_B] = BIT(DBUF_S2),
			[PIPE_C] = BIT(DBUF_S3),
			[PIPE_D] = BIT(DBUF_S4),
		},
	},
	{}
};

4698 4699 4700 4701
static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
	{
		.active_pipes = BIT(PIPE_A),
		.dbuf_mask = {
4702
			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4703
		},
4704
		.join_mbus = true,
4705 4706 4707 4708
	},
	{
		.active_pipes = BIT(PIPE_B),
		.dbuf_mask = {
4709
			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4710
		},
4711
		.join_mbus = true,
4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
		},
	},
	{
		.active_pipes = BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
		},
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
		},
	},
	{
		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
		},
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
		},
	},
	{
		.active_pipes = BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
		},
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
		},
	},
	{
		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
		},
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
		},
	},
	{
		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
		},
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
		},
	},
	{
		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
		},
	},
	{
		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
		.dbuf_mask = {
			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
		},
	},
	{}

};

4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828
static bool check_mbus_joined(u8 active_pipes,
			      const struct dbuf_slice_conf_entry *dbuf_slices)
{
	int i;

	for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
		if (dbuf_slices[i].active_pipes == active_pipes)
			return dbuf_slices[i].join_mbus;
	}
	return false;
}

static bool adlp_check_mbus_joined(u8 active_pipes)
{
	return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
}

4829 4830
static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
			      const struct dbuf_slice_conf_entry *dbuf_slices)
4831 4832 4833
{
	int i;

4834
	for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845
		if (dbuf_slices[i].active_pipes == active_pipes)
			return dbuf_slices[i].dbuf_mask[pipe];
	}
	return 0;
}

/*
 * This function finds an entry with same enabled pipe configuration and
 * returns correspondent DBuf slice mask as stated in BSpec for particular
 * platform.
 */
4846
static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859
{
	/*
	 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
	 * required calculating "pipe ratio" in order to determine
	 * if one or two slices can be used for single pipe configurations
	 * as additional constraint to the existing table.
	 * However based on recent info, it should be not "pipe ratio"
	 * but rather ratio between pixel_rate and cdclk with additional
	 * constants, so for now we are using only table until this is
	 * clarified. Also this is the reason why crtc_state param is
	 * still here - we will need it once those additional constraints
	 * pop up.
	 */
4860
	return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
4861 4862
}

4863
static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4864
{
4865
	return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
4866 4867
}

4868 4869 4870 4871 4872
static u32 adlp_compute_dbuf_slices(enum pipe pipe, u32 active_pipes)
{
	return compute_dbuf_slices(pipe, active_pipes, adlp_allowed_dbufs);
}

4873 4874 4875 4876 4877
static u32 dg2_compute_dbuf_slices(enum pipe pipe, u32 active_pipes)
{
	return compute_dbuf_slices(pipe, active_pipes, dg2_allowed_dbufs);
}

4878
static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes)
4879 4880 4881 4882
{
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	enum pipe pipe = crtc->pipe;

4883 4884 4885
	if (IS_DG2(dev_priv))
		return dg2_compute_dbuf_slices(pipe, active_pipes);
	else if (IS_ALDERLAKE_P(dev_priv))
4886 4887
		return adlp_compute_dbuf_slices(pipe, active_pipes);
	else if (DISPLAY_VER(dev_priv) == 12)
4888
		return tgl_compute_dbuf_slices(pipe, active_pipes);
4889
	else if (DISPLAY_VER(dev_priv) == 11)
4890
		return icl_compute_dbuf_slices(pipe, active_pipes);
4891 4892 4893 4894
	/*
	 * For anything else just return one slice yet.
	 * Should be extended for other platforms.
	 */
4895
	return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4896 4897
}

4898
static u64
4899 4900
skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
			     const struct intel_plane_state *plane_state,
4901
			     int color_plane)
4902
{
4903
	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4904
	const struct drm_framebuffer *fb = plane_state->hw.fb;
4905 4906
	u32 data_rate;
	u32 width = 0, height = 0;
4907
	uint_fixed_16_16_t down_scale_amount;
4908
	u64 rate;
4909

4910
	if (!plane_state->uapi.visible)
4911
		return 0;
4912

4913
	if (plane->id == PLANE_CURSOR)
4914
		return 0;
4915 4916

	if (color_plane == 1 &&
4917
	    !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4918
		return 0;
4919

4920 4921 4922 4923 4924
	/*
	 * Src coordinates are already rotated by 270 degrees for
	 * the 90/270 degree plane rotation cases (to match the
	 * GTT mapping), hence no need to account for rotation here.
	 */
4925 4926
	width = drm_rect_width(&plane_state->uapi.src) >> 16;
	height = drm_rect_height(&plane_state->uapi.src) >> 16;
4927

4928
	/* UV plane does 1/2 pixel sub-sampling */
4929
	if (color_plane == 1) {
4930 4931
		width /= 2;
		height /= 2;
4932 4933
	}

4934
	data_rate = width * height;
4935

4936
	down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4937

4938 4939
	rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);

4940
	rate *= fb->format->cpp[color_plane];
4941
	return rate;
4942 4943
}

4944
static u64
4945 4946
skl_get_total_relative_data_rate(struct intel_atomic_state *state,
				 struct intel_crtc *crtc)
4947
{
4948 4949
	struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
4950
	const struct intel_plane_state *plane_state;
4951
	struct intel_plane *plane;
4952
	u64 total_data_rate = 0;
4953 4954
	enum plane_id plane_id;
	int i;
4955

4956
	/* Calculate and cache data rate for each plane */
4957 4958 4959 4960 4961
	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
		if (plane->pipe != crtc->pipe)
			continue;

		plane_id = plane->id;
4962

4963
		/* packed/y */
4964 4965
		crtc_state->plane_data_rate[plane_id] =
			skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4966

4967
		/* uv-plane */
4968 4969 4970 4971 4972 4973 4974
		crtc_state->uv_plane_data_rate[plane_id] =
			skl_plane_relative_data_rate(crtc_state, plane_state, 1);
	}

	for_each_plane_id_on_crtc(crtc, plane_id) {
		total_data_rate += crtc_state->plane_data_rate[plane_id];
		total_data_rate += crtc_state->uv_plane_data_rate[plane_id];
4975 4976 4977 4978 4979
	}

	return total_data_rate;
}

4980
static u64
4981 4982
icl_get_total_relative_data_rate(struct intel_atomic_state *state,
				 struct intel_crtc *crtc)
4983
{
4984 4985
	struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
4986
	const struct intel_plane_state *plane_state;
4987
	struct intel_plane *plane;
4988
	u64 total_data_rate = 0;
4989 4990
	enum plane_id plane_id;
	int i;
4991 4992

	/* Calculate and cache data rate for each plane */
4993 4994 4995 4996 4997
	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
		if (plane->pipe != crtc->pipe)
			continue;

		plane_id = plane->id;
4998

4999
		if (!plane_state->planar_linked_plane) {
5000 5001
			crtc_state->plane_data_rate[plane_id] =
				skl_plane_relative_data_rate(crtc_state, plane_state, 0);
5002 5003 5004 5005 5006
		} else {
			enum plane_id y_plane_id;

			/*
			 * The slave plane might not iterate in
5007
			 * intel_atomic_crtc_state_for_each_plane_state(),
5008 5009 5010 5011
			 * and needs the master plane state which may be
			 * NULL if we try get_new_plane_state(), so we
			 * always calculate from the master.
			 */
5012
			if (plane_state->planar_slave)
5013 5014 5015
				continue;

			/* Y plane rate is calculated on the slave */
5016
			y_plane_id = plane_state->planar_linked_plane->id;
5017 5018
			crtc_state->plane_data_rate[y_plane_id] =
				skl_plane_relative_data_rate(crtc_state, plane_state, 0);
5019

5020 5021
			crtc_state->plane_data_rate[plane_id] =
				skl_plane_relative_data_rate(crtc_state, plane_state, 1);
5022 5023 5024
		}
	}

5025 5026 5027
	for_each_plane_id_on_crtc(crtc, plane_id)
		total_data_rate += crtc_state->plane_data_rate[plane_id];

5028 5029 5030
	return total_data_rate;
}

5031
const struct skl_wm_level *
5032
skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
5033 5034 5035
		   enum plane_id plane_id,
		   int level)
{
5036 5037 5038
	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];

	if (level == 0 && pipe_wm->use_sagv_wm)
5039
		return &wm->sagv.wm0;
5040 5041 5042 5043

	return &wm->wm[level];
}

5044
const struct skl_wm_level *
5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055
skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
		   enum plane_id plane_id)
{
	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];

	if (pipe_wm->use_sagv_wm)
		return &wm->sagv.trans_wm;

	return &wm->trans_wm;
}

5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085
/*
 * We only disable the watermarks for each plane if
 * they exceed the ddb allocation of said plane. This
 * is done so that we don't end up touching cursor
 * watermarks needlessly when some other plane reduces
 * our max possible watermark level.
 *
 * Bspec has this to say about the PLANE_WM enable bit:
 * "All the watermarks at this level for all enabled
 *  planes must be enabled before the level will be used."
 * So this is actually safe to do.
 */
static void
skl_check_wm_level(struct skl_wm_level *wm, u64 total)
{
	if (wm->min_ddb_alloc > total)
		memset(wm, 0, sizeof(*wm));
}

static void
skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
			u64 total, u64 uv_total)
{
	if (wm->min_ddb_alloc > total ||
	    uv_wm->min_ddb_alloc > uv_total) {
		memset(wm, 0, sizeof(*wm));
		memset(uv_wm, 0, sizeof(*uv_wm));
	}
}

5086
static int
5087 5088
skl_allocate_plane_ddb(struct intel_atomic_state *state,
		       struct intel_crtc *crtc)
5089
{
5090
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5091 5092
	struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
5093
	const struct intel_dbuf_state *dbuf_state =
5094
		intel_atomic_get_new_dbuf_state(state);
5095 5096
	const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
	int num_active = hweight8(dbuf_state->active_pipes);
5097 5098 5099
	u16 alloc_size, start = 0;
	u16 total[I915_MAX_PLANES] = {};
	u16 uv_total[I915_MAX_PLANES] = {};
5100
	u64 total_data_rate;
5101
	enum plane_id plane_id;
5102
	u32 blocks;
5103
	int level;
5104

5105
	/* Clear the partitioning for disabled planes. */
5106 5107
	memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
	memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
5108

5109
	if (!crtc_state->hw.active)
5110 5111
		return 0;

5112
	if (DISPLAY_VER(dev_priv) >= 11)
5113
		total_data_rate =
5114
			icl_get_total_relative_data_rate(state, crtc);
5115
	else
5116
		total_data_rate =
5117
			skl_get_total_relative_data_rate(state, crtc);
5118

5119
	alloc_size = skl_ddb_entry_size(alloc);
5120
	if (alloc_size == 0)
5121
		return 0;
5122

5123
	/* Allocate fixed number of blocks for cursor. */
5124
	total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
5125
	alloc_size -= total[PLANE_CURSOR];
5126
	crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
5127
		alloc->end - total[PLANE_CURSOR];
5128
	crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
5129 5130 5131

	if (total_data_rate == 0)
		return 0;
5132

5133
	/*
5134 5135
	 * Find the highest watermark level for which we can satisfy the block
	 * requirement of active planes.
5136
	 */
5137
	for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
5138
		blocks = 0;
5139
		for_each_plane_id_on_crtc(crtc, plane_id) {
5140
			const struct skl_plane_wm *wm =
5141
				&crtc_state->wm.skl.optimal.planes[plane_id];
5142 5143

			if (plane_id == PLANE_CURSOR) {
5144
				if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
5145 5146
					drm_WARN_ON(&dev_priv->drm,
						    wm->wm[level].min_ddb_alloc != U16_MAX);
5147 5148 5149
					blocks = U32_MAX;
					break;
				}
5150
				continue;
5151
			}
5152

5153 5154
			blocks += wm->wm[level].min_ddb_alloc;
			blocks += wm->uv_wm[level].min_ddb_alloc;
5155 5156
		}

5157
		if (blocks <= alloc_size) {
5158 5159 5160
			alloc_size -= blocks;
			break;
		}
5161 5162
	}

5163
	if (level < 0) {
5164 5165 5166 5167
		drm_dbg_kms(&dev_priv->drm,
			    "Requested display configuration exceeds system DDB limitations");
		drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
			    blocks, alloc_size);
5168 5169 5170
		return -EINVAL;
	}

5171
	/*
5172 5173 5174
	 * Grant each plane the blocks it requires at the highest achievable
	 * watermark level, plus an extra share of the leftover blocks
	 * proportional to its relative data rate.
5175
	 */
5176
	for_each_plane_id_on_crtc(crtc, plane_id) {
5177
		const struct skl_plane_wm *wm =
5178
			&crtc_state->wm.skl.optimal.planes[plane_id];
5179 5180
		u64 rate;
		u16 extra;
5181

5182
		if (plane_id == PLANE_CURSOR)
5183 5184
			continue;

5185
		/*
5186 5187
		 * We've accounted for all active planes; remaining planes are
		 * all disabled.
5188
		 */
5189 5190
		if (total_data_rate == 0)
			break;
5191

5192
		rate = crtc_state->plane_data_rate[plane_id];
5193 5194 5195
		extra = min_t(u16, alloc_size,
			      DIV64_U64_ROUND_UP(alloc_size * rate,
						 total_data_rate));
5196
		total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
5197 5198
		alloc_size -= extra;
		total_data_rate -= rate;
5199

5200 5201
		if (total_data_rate == 0)
			break;
5202

5203
		rate = crtc_state->uv_plane_data_rate[plane_id];
5204 5205 5206
		extra = min_t(u16, alloc_size,
			      DIV64_U64_ROUND_UP(alloc_size * rate,
						 total_data_rate));
5207
		uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
5208 5209 5210
		alloc_size -= extra;
		total_data_rate -= rate;
	}
5211
	drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
5212 5213 5214

	/* Set the actual DDB start/end points for each plane */
	start = alloc->start;
5215
	for_each_plane_id_on_crtc(crtc, plane_id) {
5216
		struct skl_ddb_entry *plane_alloc =
5217
			&crtc_state->wm.skl.plane_ddb_y[plane_id];
5218
		struct skl_ddb_entry *uv_plane_alloc =
5219
			&crtc_state->wm.skl.plane_ddb_uv[plane_id];
5220 5221 5222 5223

		if (plane_id == PLANE_CURSOR)
			continue;

5224
		/* Gen11+ uses a separate plane for UV watermarks */
5225
		drm_WARN_ON(&dev_priv->drm,
5226
			    DISPLAY_VER(dev_priv) >= 11 && uv_total[plane_id]);
5227 5228 5229 5230 5231 5232 5233

		/* Leave disabled planes at (0,0) */
		if (total[plane_id]) {
			plane_alloc->start = start;
			start += total[plane_id];
			plane_alloc->end = start;
		}
5234

5235 5236 5237 5238
		if (uv_total[plane_id]) {
			uv_plane_alloc->start = start;
			start += uv_total[plane_id];
			uv_plane_alloc->end = start;
5239
		}
5240
	}
5241

5242 5243 5244 5245 5246 5247 5248
	/*
	 * When we calculated watermark values we didn't know how high
	 * of a level we'd actually be able to hit, so we just marked
	 * all levels as "enabled."  Go back now and disable the ones
	 * that aren't actually possible.
	 */
	for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
5249
		for_each_plane_id_on_crtc(crtc, plane_id) {
5250
			struct skl_plane_wm *wm =
5251
				&crtc_state->wm.skl.optimal.planes[plane_id];
5252

5253 5254
			skl_check_nv12_wm_level(&wm->wm[level], &wm->uv_wm[level],
						total[plane_id], uv_total[plane_id]);
5255

5256
			/*
5257
			 * Wa_1408961008:icl, ehl
5258 5259
			 * Underruns with WM1+ disabled
			 */
5260
			if (DISPLAY_VER(dev_priv) == 11 &&
5261 5262 5263
			    level == 1 && wm->wm[0].enable) {
				wm->wm[level].blocks = wm->wm[0].blocks;
				wm->wm[level].lines = wm->wm[0].lines;
5264
				wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
5265
			}
5266 5267 5268 5269
		}
	}

	/*
5270 5271
	 * Go back and disable the transition and SAGV watermarks
	 * if it turns out we don't have enough DDB blocks for them.
5272
	 */
5273
	for_each_plane_id_on_crtc(crtc, plane_id) {
5274
		struct skl_plane_wm *wm =
5275
			&crtc_state->wm.skl.optimal.planes[plane_id];
5276

5277 5278 5279
		skl_check_wm_level(&wm->trans_wm, total[plane_id]);
		skl_check_wm_level(&wm->sagv.wm0, total[plane_id]);
		skl_check_wm_level(&wm->sagv.trans_wm, total[plane_id]);
5280 5281
	}

5282
	return 0;
5283 5284
}

5285 5286
/*
 * The max latency should be 257 (max the punit can code is 255 and we add 2us
5287
 * for the read latency) and cpp should always be <= 8, so that
5288 5289 5290
 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
*/
5291
static uint_fixed_16_16_t
5292 5293
skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
	       u8 cpp, u32 latency, u32 dbuf_block_size)
5294
{
5295
	u32 wm_intermediate_val;
5296
	uint_fixed_16_16_t ret;
5297 5298

	if (latency == 0)
5299
		return FP_16_16_MAX;
5300

5301
	wm_intermediate_val = latency * pixel_rate * cpp;
5302
	ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5303

5304
	if (DISPLAY_VER(dev_priv) >= 10)
5305 5306
		ret = add_fixed16_u32(ret, 1);

5307 5308 5309
	return ret;
}

5310 5311 5312
static uint_fixed_16_16_t
skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
	       uint_fixed_16_16_t plane_blocks_per_line)
5313
{
5314
	u32 wm_intermediate_val;
5315
	uint_fixed_16_16_t ret;
5316 5317

	if (latency == 0)
5318
		return FP_16_16_MAX;
5319 5320

	wm_intermediate_val = latency * pixel_rate;
5321 5322
	wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
					   pipe_htotal * 1000);
5323
	ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5324 5325 5326
	return ret;
}

5327
static uint_fixed_16_16_t
5328
intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5329
{
5330
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5331 5332
	u32 pixel_rate;
	u32 crtc_htotal;
5333 5334
	uint_fixed_16_16_t linetime_us;

5335
	if (!crtc_state->hw.active)
5336
		return u32_to_fixed16(0);
5337

5338
	pixel_rate = crtc_state->pixel_rate;
5339

5340
	if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5341
		return u32_to_fixed16(0);
5342

5343
	crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
5344
	linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5345 5346 5347 5348

	return linetime_us;
}

5349
static int
5350 5351 5352 5353 5354
skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
		      int width, const struct drm_format_info *format,
		      u64 modifier, unsigned int rotation,
		      u32 plane_pixel_rate, struct skl_wm_params *wp,
		      int color_plane)
5355
{
5356
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5357
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5358
	u32 interm_pbpl;
5359

5360
	/* only planar format has two planes */
5361 5362
	if (color_plane == 1 &&
	    !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5363 5364
		drm_dbg_kms(&dev_priv->drm,
			    "Non planar format have single plane\n");
5365 5366 5367
		return -EINVAL;
	}

5368 5369 5370 5371 5372 5373 5374
	wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
		      modifier == I915_FORMAT_MOD_Yf_TILED ||
		      modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
		      modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
	wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
	wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
			 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5375
	wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5376

5377
	wp->width = width;
5378
	if (color_plane == 1 && wp->is_planar)
5379 5380
		wp->width /= 2;

5381 5382
	wp->cpp = format->cpp[color_plane];
	wp->plane_pixel_rate = plane_pixel_rate;
5383

5384
	if (DISPLAY_VER(dev_priv) >= 11 &&
5385
	    modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
5386 5387 5388 5389
		wp->dbuf_block_size = 256;
	else
		wp->dbuf_block_size = 512;

5390
	if (drm_rotation_90_or_270(rotation)) {
5391
		switch (wp->cpp) {
5392
		case 1:
5393
			wp->y_min_scanlines = 16;
5394 5395
			break;
		case 2:
5396
			wp->y_min_scanlines = 8;
5397 5398
			break;
		case 4:
5399
			wp->y_min_scanlines = 4;
5400
			break;
5401
		default:
5402
			MISSING_CASE(wp->cpp);
5403
			return -EINVAL;
5404 5405
		}
	} else {
5406
		wp->y_min_scanlines = 4;
5407 5408
	}

5409
	if (skl_needs_memory_bw_wa(dev_priv))
5410
		wp->y_min_scanlines *= 2;
5411

5412 5413 5414
	wp->plane_bytes_per_line = wp->width * wp->cpp;
	if (wp->y_tiled) {
		interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5415 5416
					   wp->y_min_scanlines,
					   wp->dbuf_block_size);
5417

5418
		if (DISPLAY_VER(dev_priv) >= 10)
5419 5420
			interm_pbpl++;

5421 5422
		wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
							wp->y_min_scanlines);
5423
	} else {
5424
		interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5425 5426
					   wp->dbuf_block_size);

5427
		if (!wp->x_tiled || DISPLAY_VER(dev_priv) >= 10)
5428 5429
			interm_pbpl++;

5430
		wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5431 5432
	}

5433 5434
	wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
					     wp->plane_blocks_per_line);
5435

5436
	wp->linetime_us = fixed16_to_u32_round_up(
5437
					intel_get_linetime_us(crtc_state));
5438 5439 5440 5441

	return 0;
}

5442 5443 5444 5445 5446
static int
skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
			    const struct intel_plane_state *plane_state,
			    struct skl_wm_params *wp, int color_plane)
{
5447
	const struct drm_framebuffer *fb = plane_state->hw.fb;
5448 5449
	int width;

5450 5451 5452 5453 5454
	/*
	 * Src coordinates are already rotated by 270 degrees for
	 * the 90/270 degree plane rotation cases (to match the
	 * GTT mapping), hence no need to account for rotation here.
	 */
5455
	width = drm_rect_width(&plane_state->uapi.src) >> 16;
5456 5457 5458

	return skl_compute_wm_params(crtc_state, width,
				     fb->format, fb->modifier,
5459
				     plane_state->hw.rotation,
5460
				     intel_plane_pixel_rate(crtc_state, plane_state),
5461 5462 5463
				     wp, color_plane);
}

5464 5465
static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
{
5466
	if (DISPLAY_VER(dev_priv) >= 10)
5467 5468 5469 5470 5471 5472
		return true;

	/* The number of lines are ignored for the level 0 watermark. */
	return level > 0;
}

5473 5474 5475 5476 5477 5478 5479 5480
static int skl_wm_max_lines(struct drm_i915_private *dev_priv)
{
	if (DISPLAY_VER(dev_priv) >= 13)
		return 255;
	else
		return 31;
}

5481
static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5482
				 int level,
5483
				 unsigned int latency,
5484 5485 5486
				 const struct skl_wm_params *wp,
				 const struct skl_wm_level *result_prev,
				 struct skl_wm_level *result /* out */)
5487
{
5488
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5489 5490
	uint_fixed_16_16_t method1, method2;
	uint_fixed_16_16_t selected_result;
5491
	u32 blocks, lines, min_ddb_alloc = 0;
5492

5493 5494 5495
	if (latency == 0) {
		/* reject it */
		result->min_ddb_alloc = U16_MAX;
5496
		return;
5497
	}
5498

5499 5500 5501 5502
	/*
	 * WaIncreaseLatencyIPCEnabled: kbl,cfl
	 * Display WA #1141: kbl,cfl
	 */
5503 5504 5505
	if ((IS_KABYLAKE(dev_priv) ||
	     IS_COFFEELAKE(dev_priv) ||
	     IS_COMETLAKE(dev_priv)) &&
5506 5507 5508
	    dev_priv->ipc_enabled)
		latency += 4;

5509
	if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5510 5511 5512
		latency += 15;

	method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5513
				 wp->cpp, latency, wp->dbuf_block_size);
5514
	method2 = skl_wm_method2(wp->plane_pixel_rate,
5515
				 crtc_state->hw.pipe_mode.crtc_htotal,
5516
				 latency,
5517
				 wp->plane_blocks_per_line);
5518

5519 5520
	if (wp->y_tiled) {
		selected_result = max_fixed16(method2, wp->y_tile_minimum);
5521
	} else {
5522
		if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
5523
		     wp->dbuf_block_size < 1) &&
5524
		     (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5525
			selected_result = method2;
5526
		} else if (latency >= wp->linetime_us) {
5527
			if (DISPLAY_VER(dev_priv) == 9)
5528 5529 5530 5531
				selected_result = min_fixed16(method1, method2);
			else
				selected_result = method2;
		} else {
5532
			selected_result = method1;
5533
		}
5534
	}
5535

5536 5537 5538
	blocks = fixed16_to_u32_round_up(selected_result) + 1;
	lines = div_round_up_fixed16(selected_result,
				     wp->plane_blocks_per_line);
5539

5540
	if (DISPLAY_VER(dev_priv) == 9) {
5541 5542
		/* Display WA #1125: skl,bxt,kbl */
		if (level == 0 && wp->rc_surface)
5543
			blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5544 5545 5546 5547

		/* Display WA #1126: skl,bxt,kbl */
		if (level >= 1 && level <= 7) {
			if (wp->y_tiled) {
5548 5549
				blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
				lines += wp->y_min_scanlines;
5550
			} else {
5551
				blocks++;
5552
			}
5553

5554 5555 5556 5557 5558 5559
			/*
			 * Make sure result blocks for higher latency levels are
			 * atleast as high as level below the current level.
			 * Assumption in DDB algorithm optimization for special
			 * cases. Also covers Display WA #1125 for RC.
			 */
5560 5561
			if (result_prev->blocks > blocks)
				blocks = result_prev->blocks;
5562
		}
5563
	}
5564

5565
	if (DISPLAY_VER(dev_priv) >= 11) {
5566 5567 5568
		if (wp->y_tiled) {
			int extra_lines;

5569
			if (lines % wp->y_min_scanlines == 0)
5570 5571 5572
				extra_lines = wp->y_min_scanlines;
			else
				extra_lines = wp->y_min_scanlines * 2 -
5573
					lines % wp->y_min_scanlines;
5574

5575
			min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
5576 5577
								 wp->plane_blocks_per_line);
		} else {
5578
			min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
5579 5580 5581
		}
	}

5582
	if (!skl_wm_has_lines(dev_priv, level))
5583
		lines = 0;
5584

5585
	if (lines > skl_wm_max_lines(dev_priv)) {
5586 5587
		/* reject it */
		result->min_ddb_alloc = U16_MAX;
5588
		return;
5589
	}
5590 5591

	/*
5592
	 * If lines is valid, assume we can use this watermark level
5593 5594 5595 5596
	 * for now.  We'll come back and disable it after we calculate the
	 * DDB allocation if it turns out we don't actually have enough
	 * blocks to satisfy it.
	 */
5597 5598
	result->blocks = blocks;
	result->lines = lines;
5599
	/* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5600 5601
	result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
	result->enable = true;
5602

5603
	if (DISPLAY_VER(dev_priv) < 12)
5604
		result->can_sagv = latency >= dev_priv->sagv_block_time_us;
5605 5606
}

5607
static void
5608
skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5609
		      const struct skl_wm_params *wm_params,
5610
		      struct skl_wm_level *levels)
5611
{
5612
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5613
	int level, max_level = ilk_wm_max_level(dev_priv);
5614
	struct skl_wm_level *result_prev = &levels[0];
5615

5616
	for (level = 0; level <= max_level; level++) {
5617
		struct skl_wm_level *result = &levels[level];
5618
		unsigned int latency = dev_priv->wm.skl_latency[level];
5619

5620 5621
		skl_compute_plane_wm(crtc_state, level, latency,
				     wm_params, result_prev, result);
5622 5623

		result_prev = result;
5624
	}
5625 5626
}

5627 5628 5629 5630 5631
static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
				const struct skl_wm_params *wm_params,
				struct skl_plane_wm *plane_wm)
{
	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5632
	struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
5633 5634 5635 5636 5637 5638 5639 5640
	struct skl_wm_level *levels = plane_wm->wm;
	unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;

	skl_compute_plane_wm(crtc_state, 0, latency,
			     wm_params, &levels[0],
			     sagv_wm);
}

5641 5642 5643 5644
static void skl_compute_transition_wm(struct drm_i915_private *dev_priv,
				      struct skl_wm_level *trans_wm,
				      const struct skl_wm_level *wm0,
				      const struct skl_wm_params *wp)
5645
{
5646
	u16 trans_min, trans_amount, trans_y_tile_min;
5647
	u16 wm0_blocks, trans_offset, blocks;
5648 5649 5650

	/* Transition WM don't make any sense if ipc is disabled */
	if (!dev_priv->ipc_enabled)
5651
		return;
5652

5653 5654 5655 5656
	/*
	 * WaDisableTWM:skl,kbl,cfl,bxt
	 * Transition WM are not recommended by HW team for GEN9
	 */
5657
	if (DISPLAY_VER(dev_priv) == 9)
5658 5659
		return;

5660
	if (DISPLAY_VER(dev_priv) >= 11)
5661
		trans_min = 4;
5662 5663 5664 5665
	else
		trans_min = 14;

	/* Display WA #1140: glk,cnl */
5666
	if (DISPLAY_VER(dev_priv) == 10)
5667 5668 5669
		trans_amount = 0;
	else
		trans_amount = 10; /* This is configurable amount */
5670

5671
	trans_offset = trans_min + trans_amount;
5672

5673 5674 5675
	/*
	 * The spec asks for Selected Result Blocks for wm0 (the real value),
	 * not Result Blocks (the integer value). Pay attention to the capital
5676
	 * letters. The value wm_l0->blocks is actually Result Blocks, but
5677 5678 5679 5680 5681 5682
	 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
	 * and since we later will have to get the ceiling of the sum in the
	 * transition watermarks calculation, we can just pretend Selected
	 * Result Blocks is Result Blocks minus 1 and it should work for the
	 * current platforms.
	 */
5683
	wm0_blocks = wm0->blocks - 1;
5684

5685
	if (wp->y_tiled) {
5686 5687
		trans_y_tile_min =
			(u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5688
		blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
5689
	} else {
5690
		blocks = wm0_blocks + trans_offset;
5691
	}
5692
	blocks++;
5693

5694 5695 5696 5697 5698
	/*
	 * Just assume we can enable the transition watermark.  After
	 * computing the DDB we'll come back and disable it if that
	 * assumption turns out to be false.
	 */
5699 5700 5701
	trans_wm->blocks = blocks;
	trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
	trans_wm->enable = true;
5702 5703
}

5704
static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5705 5706
				     const struct intel_plane_state *plane_state,
				     enum plane_id plane_id, int color_plane)
5707
{
5708 5709
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5710
	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5711 5712 5713
	struct skl_wm_params wm_params;
	int ret;

5714
	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5715 5716 5717 5718
					  &wm_params, color_plane);
	if (ret)
		return ret;

5719
	skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5720

5721 5722 5723
	skl_compute_transition_wm(dev_priv, &wm->trans_wm,
				  &wm->wm[0], &wm_params);

5724
	if (DISPLAY_VER(dev_priv) >= 12) {
5725 5726
		tgl_compute_sagv_wm(crtc_state, &wm_params, wm);

5727 5728 5729
		skl_compute_transition_wm(dev_priv, &wm->sagv.trans_wm,
					  &wm->sagv.wm0, &wm_params);
	}
5730 5731 5732 5733

	return 0;
}

5734
static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5735 5736
				 const struct intel_plane_state *plane_state,
				 enum plane_id plane_id)
5737
{
5738
	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5739 5740 5741
	struct skl_wm_params wm_params;
	int ret;

5742
	wm->is_planar = true;
5743 5744

	/* uv plane watermarks must also be validated for NV12/Planar */
5745
	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5746 5747 5748
					  &wm_params, 1);
	if (ret)
		return ret;
5749

5750
	skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5751

5752
	return 0;
5753 5754
}

5755
static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5756
			      const struct intel_plane_state *plane_state)
5757
{
5758
	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5759
	enum plane_id plane_id = plane->id;
5760 5761
	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
	const struct drm_framebuffer *fb = plane_state->hw.fb;
5762 5763
	int ret;

5764 5765
	memset(wm, 0, sizeof(*wm));

5766 5767 5768
	if (!intel_wm_plane_visible(crtc_state, plane_state))
		return 0;

5769
	ret = skl_build_plane_wm_single(crtc_state, plane_state,
5770
					plane_id, 0);
5771 5772 5773
	if (ret)
		return ret;

5774
	if (fb->format->is_yuv && fb->format->num_planes > 1) {
5775
		ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5776 5777 5778 5779 5780 5781 5782 5783
					    plane_id);
		if (ret)
			return ret;
	}

	return 0;
}

5784
static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5785 5786
			      const struct intel_plane_state *plane_state)
{
5787 5788 5789 5790
	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
	enum plane_id plane_id = plane->id;
	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5791 5792 5793
	int ret;

	/* Watermarks calculated in master */
5794
	if (plane_state->planar_slave)
5795 5796
		return 0;

5797 5798
	memset(wm, 0, sizeof(*wm));

5799
	if (plane_state->planar_linked_plane) {
5800
		const struct drm_framebuffer *fb = plane_state->hw.fb;
5801
		enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
5802

5803 5804 5805 5806
		drm_WARN_ON(&dev_priv->drm,
			    !intel_wm_plane_visible(crtc_state, plane_state));
		drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
			    fb->format->num_planes == 1);
5807

5808
		ret = skl_build_plane_wm_single(crtc_state, plane_state,
5809 5810 5811 5812
						y_plane_id, 0);
		if (ret)
			return ret;

5813
		ret = skl_build_plane_wm_single(crtc_state, plane_state,
5814 5815 5816 5817
						plane_id, 1);
		if (ret)
			return ret;
	} else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5818
		ret = skl_build_plane_wm_single(crtc_state, plane_state,
5819 5820 5821 5822 5823 5824
						plane_id, 0);
		if (ret)
			return ret;
	}

	return 0;
5825 5826
}

5827 5828
static int skl_build_pipe_wm(struct intel_atomic_state *state,
			     struct intel_crtc *crtc)
5829
{
5830 5831 5832
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
5833
	const struct intel_plane_state *plane_state;
5834 5835
	struct intel_plane *plane;
	int ret, i;
5836

5837 5838 5839 5840 5841 5842 5843 5844
	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
		/*
		 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
		 * instead but we don't populate that correctly for NV12 Y
		 * planes so for now hack this.
		 */
		if (plane->pipe != crtc->pipe)
			continue;
5845

5846
		if (DISPLAY_VER(dev_priv) >= 11)
5847
			ret = icl_build_plane_wm(crtc_state, plane_state);
5848
		else
5849
			ret = skl_build_plane_wm(crtc_state, plane_state);
5850 5851
		if (ret)
			return ret;
5852
	}
5853

5854 5855
	crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;

5856
	return 0;
5857 5858
}

5859 5860
static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
				i915_reg_t reg,
5861 5862 5863
				const struct skl_ddb_entry *entry)
{
	if (entry->end)
5864 5865
		intel_de_write_fw(dev_priv, reg,
				  (entry->end - 1) << 16 | entry->start);
5866
	else
5867
		intel_de_write_fw(dev_priv, reg, 0);
5868 5869
}

5870 5871 5872 5873
static void skl_write_wm_level(struct drm_i915_private *dev_priv,
			       i915_reg_t reg,
			       const struct skl_wm_level *level)
{
5874
	u32 val = 0;
5875

5876
	if (level->enable)
5877
		val |= PLANE_WM_EN;
5878 5879
	if (level->ignore_lines)
		val |= PLANE_WM_IGNORE_LINES;
5880
	val |= level->blocks;
5881
	val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
5882

5883
	intel_de_write_fw(dev_priv, reg, val);
5884 5885
}

5886 5887
void skl_write_plane_wm(struct intel_plane *plane,
			const struct intel_crtc_state *crtc_state)
5888
{
5889
	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5890
	int level, max_level = ilk_wm_max_level(dev_priv);
5891 5892
	enum plane_id plane_id = plane->id;
	enum pipe pipe = plane->pipe;
5893 5894
	const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5895 5896 5897 5898
	const struct skl_ddb_entry *ddb_y =
		&crtc_state->wm.skl.plane_ddb_y[plane_id];
	const struct skl_ddb_entry *ddb_uv =
		&crtc_state->wm.skl.plane_ddb_uv[plane_id];
5899

5900
	for (level = 0; level <= max_level; level++)
5901
		skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5902 5903
				   skl_plane_wm_level(pipe_wm, plane_id, level));

5904
	skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5905
			   skl_plane_trans_wm(pipe_wm, plane_id));
5906

5907 5908 5909 5910 5911 5912 5913
	if (HAS_HW_SAGV_WM(dev_priv)) {
		skl_write_wm_level(dev_priv, PLANE_WM_SAGV(pipe, plane_id),
				   &wm->sagv.wm0);
		skl_write_wm_level(dev_priv, PLANE_WM_SAGV_TRANS(pipe, plane_id),
				   &wm->sagv.trans_wm);
	}

5914
	if (DISPLAY_VER(dev_priv) >= 11) {
5915
		skl_ddb_entry_write(dev_priv,
5916 5917
				    PLANE_BUF_CFG(pipe, plane_id), ddb_y);
		return;
5918
	}
5919 5920 5921 5922 5923 5924 5925 5926

	if (wm->is_planar)
		swap(ddb_y, ddb_uv);

	skl_ddb_entry_write(dev_priv,
			    PLANE_BUF_CFG(pipe, plane_id), ddb_y);
	skl_ddb_entry_write(dev_priv,
			    PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5927 5928
}

5929 5930
void skl_write_cursor_wm(struct intel_plane *plane,
			 const struct intel_crtc_state *crtc_state)
5931
{
5932
	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5933
	int level, max_level = ilk_wm_max_level(dev_priv);
5934 5935
	enum plane_id plane_id = plane->id;
	enum pipe pipe = plane->pipe;
5936
	const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5937 5938
	const struct skl_ddb_entry *ddb =
		&crtc_state->wm.skl.plane_ddb_y[plane_id];
5939

5940
	for (level = 0; level <= max_level; level++)
5941
		skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5942 5943
				   skl_plane_wm_level(pipe_wm, plane_id, level));

5944 5945
	skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe),
			   skl_plane_trans_wm(pipe_wm, plane_id));
5946

5947 5948 5949 5950 5951 5952 5953 5954 5955
	if (HAS_HW_SAGV_WM(dev_priv)) {
		const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];

		skl_write_wm_level(dev_priv, CUR_WM_SAGV(pipe),
				   &wm->sagv.wm0);
		skl_write_wm_level(dev_priv, CUR_WM_SAGV_TRANS(pipe),
				   &wm->sagv.trans_wm);
	}

5956
	skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5957 5958
}

5959 5960 5961
bool skl_wm_level_equals(const struct skl_wm_level *l1,
			 const struct skl_wm_level *l2)
{
5962
	return l1->enable == l2->enable &&
5963
		l1->ignore_lines == l2->ignore_lines &&
5964 5965
		l1->lines == l2->lines &&
		l1->blocks == l2->blocks;
5966
}
5967

5968 5969 5970 5971 5972
static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
				const struct skl_plane_wm *wm1,
				const struct skl_plane_wm *wm2)
{
	int level, max_level = ilk_wm_max_level(dev_priv);
5973

5974
	for (level = 0; level <= max_level; level++) {
5975 5976 5977 5978 5979 5980
		/*
		 * We don't check uv_wm as the hardware doesn't actually
		 * use it. It only gets used for calculating the required
		 * ddb allocation.
		 */
		if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5981 5982 5983
			return false;
	}

5984
	return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
5985 5986
		skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
		skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
5987 5988
}

5989 5990
static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
				    const struct skl_ddb_entry *b)
5991
{
5992
	return a->start < b->end && b->start < a->end;
5993 5994
}

5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006
static void skl_ddb_entry_union(struct skl_ddb_entry *a,
				const struct skl_ddb_entry *b)
{
	if (a->end && b->end) {
		a->start = min(a->start, b->start);
		a->end = max(a->end, b->end);
	} else if (b->end) {
		a->start = b->start;
		a->end = b->end;
	}
}

6007
bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
6008
				 const struct skl_ddb_entry *entries,
6009
				 int num_entries, int ignore_idx)
6010
{
6011
	int i;
6012

6013 6014 6015
	for (i = 0; i < num_entries; i++) {
		if (i != ignore_idx &&
		    skl_ddb_entries_overlap(ddb, &entries[i]))
6016
			return true;
6017
	}
6018

6019
	return false;
6020 6021
}

6022
static int
6023 6024
skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
			    struct intel_crtc_state *new_crtc_state)
6025
{
6026 6027
	struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6028 6029
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	struct intel_plane *plane;
6030

6031 6032 6033
	for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
		struct intel_plane_state *plane_state;
		enum plane_id plane_id = plane->id;
6034

6035 6036 6037 6038
		if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
					&new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
		    skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
					&new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
6039 6040
			continue;

6041
		plane_state = intel_atomic_get_plane_state(state, plane);
6042 6043
		if (IS_ERR(plane_state))
			return PTR_ERR(plane_state);
6044

6045
		new_crtc_state->update_planes |= BIT(plane_id);
6046 6047 6048 6049 6050
	}

	return 0;
}

6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 6067 6068
static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
{
	struct drm_i915_private *dev_priv = to_i915(dbuf_state->base.state->base.dev);
	u8 enabled_slices;
	enum pipe pipe;

	/*
	 * FIXME: For now we always enable slice S1 as per
	 * the Bspec display initialization sequence.
	 */
	enabled_slices = BIT(DBUF_S1);

	for_each_pipe(dev_priv, pipe)
		enabled_slices |= dbuf_state->slices[pipe];

	return enabled_slices;
}

6069
static int
6070
skl_compute_ddb(struct intel_atomic_state *state)
6071
{
6072 6073
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
	const struct intel_dbuf_state *old_dbuf_state;
6074
	struct intel_dbuf_state *new_dbuf_state = NULL;
6075
	const struct intel_crtc_state *old_crtc_state;
6076
	struct intel_crtc_state *new_crtc_state;
6077 6078
	struct intel_crtc *crtc;
	int ret, i;
6079

6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096
	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
		new_dbuf_state = intel_atomic_get_dbuf_state(state);
		if (IS_ERR(new_dbuf_state))
			return PTR_ERR(new_dbuf_state);

		old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
		break;
	}

	if (!new_dbuf_state)
		return 0;

	new_dbuf_state->active_pipes =
		intel_calc_active_pipes(state, old_dbuf_state->active_pipes);

	if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6097 6098
		if (ret)
			return ret;
6099
	}
6100

6101 6102 6103 6104 6105 6106 6107 6108 6109 6110
	for_each_intel_crtc(&dev_priv->drm, crtc) {
		enum pipe pipe = crtc->pipe;

		new_dbuf_state->slices[pipe] =
			skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes);

		if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
			continue;

		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6111 6112
		if (ret)
			return ret;
6113 6114
	}

6115 6116
	new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);

6117 6118 6119 6120 6121
	if (IS_ALDERLAKE_P(dev_priv))
		new_dbuf_state->joined_mbus = adlp_check_mbus_joined(new_dbuf_state->active_pipes);

	if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
	    old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6122 6123 6124
		ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
		if (ret)
			return ret;
6125

6126 6127 6128 6129 6130 6131 6132
		if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
			/* TODO: Implement vblank synchronized MBUS joining changes */
			ret = intel_modeset_all_pipes(state);
			if (ret)
				return ret;
		}

6133
		drm_dbg_kms(&dev_priv->drm,
6134
			    "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
6135 6136
			    old_dbuf_state->enabled_slices,
			    new_dbuf_state->enabled_slices,
6137 6138 6139
			    INTEL_INFO(dev_priv)->dbuf.slice_mask,
			    yesno(old_dbuf_state->joined_mbus),
			    yesno(new_dbuf_state->joined_mbus));
6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171
	}

	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
		enum pipe pipe = crtc->pipe;

		new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);

		if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
			continue;

		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
		if (ret)
			return ret;
	}

	for_each_intel_crtc(&dev_priv->drm, crtc) {
		ret = skl_crtc_allocate_ddb(state, crtc);
		if (ret)
			return ret;
	}

	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
					    new_crtc_state, i) {
		ret = skl_allocate_plane_ddb(state, crtc);
		if (ret)
			return ret;

		ret = skl_ddb_add_affected_planes(old_crtc_state,
						  new_crtc_state);
		if (ret)
			return ret;
	}
6172

6173 6174 6175
	return 0;
}

6176 6177 6178 6179 6180
static char enast(bool enable)
{
	return enable ? '*' : ' ';
}

6181
static void
6182
skl_print_wm_changes(struct intel_atomic_state *state)
6183
{
6184 6185 6186 6187 6188
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
	const struct intel_crtc_state *old_crtc_state;
	const struct intel_crtc_state *new_crtc_state;
	struct intel_plane *plane;
	struct intel_crtc *crtc;
6189
	int i;
6190

6191
	if (!drm_debug_enabled(DRM_UT_KMS))
6192 6193
		return;

6194 6195
	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
					    new_crtc_state, i) {
6196 6197 6198 6199 6200
		const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;

		old_pipe_wm = &old_crtc_state->wm.skl.optimal;
		new_pipe_wm = &new_crtc_state->wm.skl.optimal;

6201 6202
		for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
			enum plane_id plane_id = plane->id;
6203 6204
			const struct skl_ddb_entry *old, *new;

6205 6206
			old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
			new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
6207 6208 6209 6210

			if (skl_ddb_entry_equal(old, new))
				continue;

6211 6212 6213 6214 6215
			drm_dbg_kms(&dev_priv->drm,
				    "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
				    plane->base.base.id, plane->base.name,
				    old->start, old->end, new->start, new->end,
				    skl_ddb_entry_size(old), skl_ddb_entry_size(new));
6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227
		}

		for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
			enum plane_id plane_id = plane->id;
			const struct skl_plane_wm *old_wm, *new_wm;

			old_wm = &old_pipe_wm->planes[plane_id];
			new_wm = &new_pipe_wm->planes[plane_id];

			if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
				continue;

6228
			drm_dbg_kms(&dev_priv->drm,
6229 6230
				    "[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
				    " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
6231
				    plane->base.base.id, plane->base.name,
6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245
				    enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
				    enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
				    enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
				    enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
				    enast(old_wm->trans_wm.enable),
				    enast(old_wm->sagv.wm0.enable),
				    enast(old_wm->sagv.trans_wm.enable),
				    enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
				    enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
				    enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
				    enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
				    enast(new_wm->trans_wm.enable),
				    enast(new_wm->sagv.wm0.enable),
				    enast(new_wm->sagv.trans_wm.enable));
6246 6247

			drm_dbg_kms(&dev_priv->drm,
6248 6249
				    "[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
				      " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
6250
				    plane->base.base.id, plane->base.name,
6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272
				    enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
				    enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
				    enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
				    enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
				    enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
				    enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
				    enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
				    enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
				    enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
				    enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
				    enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
				    enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
				    enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
				    enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
				    enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
				    enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
				    enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
				    enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
				    enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
				    enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
				    enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
				    enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
6273 6274

			drm_dbg_kms(&dev_priv->drm,
6275 6276
				    "[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
				    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6277
				    plane->base.base.id, plane->base.name,
6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291
				    old_wm->wm[0].blocks, old_wm->wm[1].blocks,
				    old_wm->wm[2].blocks, old_wm->wm[3].blocks,
				    old_wm->wm[4].blocks, old_wm->wm[5].blocks,
				    old_wm->wm[6].blocks, old_wm->wm[7].blocks,
				    old_wm->trans_wm.blocks,
				    old_wm->sagv.wm0.blocks,
				    old_wm->sagv.trans_wm.blocks,
				    new_wm->wm[0].blocks, new_wm->wm[1].blocks,
				    new_wm->wm[2].blocks, new_wm->wm[3].blocks,
				    new_wm->wm[4].blocks, new_wm->wm[5].blocks,
				    new_wm->wm[6].blocks, new_wm->wm[7].blocks,
				    new_wm->trans_wm.blocks,
				    new_wm->sagv.wm0.blocks,
				    new_wm->sagv.trans_wm.blocks);
6292 6293

			drm_dbg_kms(&dev_priv->drm,
6294 6295
				    "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
				    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6296 6297 6298 6299 6300 6301
				    plane->base.base.id, plane->base.name,
				    old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
				    old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
				    old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
				    old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
				    old_wm->trans_wm.min_ddb_alloc,
6302
				    old_wm->sagv.wm0.min_ddb_alloc,
6303
				    old_wm->sagv.trans_wm.min_ddb_alloc,
6304 6305 6306 6307
				    new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
				    new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
				    new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
				    new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
6308
				    new_wm->trans_wm.min_ddb_alloc,
6309 6310
				    new_wm->sagv.wm0.min_ddb_alloc,
				    new_wm->sagv.trans_wm.min_ddb_alloc);
6311 6312 6313 6314
		}
	}
}

6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327
static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
					 const struct skl_pipe_wm *old_pipe_wm,
					 const struct skl_pipe_wm *new_pipe_wm)
{
	struct drm_i915_private *i915 = to_i915(plane->base.dev);
	int level, max_level = ilk_wm_max_level(i915);

	for (level = 0; level <= max_level; level++) {
		/*
		 * We don't check uv_wm as the hardware doesn't actually
		 * use it. It only gets used for calculating the required
		 * ddb allocation.
		 */
6328 6329
		if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
					 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
6330 6331 6332
			return false;
	}

6333 6334 6335 6336 6337 6338 6339 6340 6341
	if (HAS_HW_SAGV_WM(i915)) {
		const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
		const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];

		if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
		    !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
			return false;
	}

6342 6343
	return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
				   skl_plane_trans_wm(new_pipe_wm, plane->id));
6344 6345
}

6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364 6365 6366 6367 6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389
/*
 * To make sure the cursor watermark registers are always consistent
 * with our computed state the following scenario needs special
 * treatment:
 *
 * 1. enable cursor
 * 2. move cursor entirely offscreen
 * 3. disable cursor
 *
 * Step 2. does call .disable_plane() but does not zero the watermarks
 * (since we consider an offscreen cursor still active for the purposes
 * of watermarks). Step 3. would not normally call .disable_plane()
 * because the actual plane visibility isn't changing, and we don't
 * deallocate the cursor ddb until the pipe gets disabled. So we must
 * force step 3. to call .disable_plane() to update the watermark
 * registers properly.
 *
 * Other planes do not suffer from this issues as their watermarks are
 * calculated based on the actual plane visibility. The only time this
 * can trigger for the other planes is during the initial readout as the
 * default value of the watermarks registers is not zero.
 */
static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
				      struct intel_crtc *crtc)
{
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	const struct intel_crtc_state *old_crtc_state =
		intel_atomic_get_old_crtc_state(state, crtc);
	struct intel_crtc_state *new_crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
	struct intel_plane *plane;

	for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
		struct intel_plane_state *plane_state;
		enum plane_id plane_id = plane->id;

		/*
		 * Force a full wm update for every plane on modeset.
		 * Required because the reset value of the wm registers
		 * is non-zero, whereas we want all disabled planes to
		 * have zero watermarks. So if we turn off the relevant
		 * power well the hardware state will go out of sync
		 * with the software state.
		 */
6390
		if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6391 6392 6393
		    skl_plane_selected_wm_equals(plane,
						 &old_crtc_state->wm.skl.optimal,
						 &new_crtc_state->wm.skl.optimal))
6394 6395 6396 6397 6398 6399 6400 6401 6402 6403 6404 6405
			continue;

		plane_state = intel_atomic_get_plane_state(state, plane);
		if (IS_ERR(plane_state))
			return PTR_ERR(plane_state);

		new_crtc_state->update_planes |= BIT(plane_id);
	}

	return 0;
}

6406
static int
6407
skl_compute_wm(struct intel_atomic_state *state)
6408
{
6409
	struct intel_crtc *crtc;
6410
	struct intel_crtc_state *new_crtc_state;
6411 6412
	int ret, i;

6413 6414
	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
		ret = skl_build_pipe_wm(state, crtc);
6415 6416
		if (ret)
			return ret;
6417 6418
	}

6419 6420 6421 6422
	ret = skl_compute_ddb(state);
	if (ret)
		return ret;

6423 6424 6425
	ret = intel_compute_sagv_mask(state);
	if (ret)
		return ret;
6426

6427 6428 6429 6430 6431
	/*
	 * skl_compute_ddb() will have adjusted the final watermarks
	 * based on how much ddb is available. Now we can actually
	 * check if the final watermarks changed.
	 */
6432
	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6433 6434 6435 6436 6437
		ret = skl_wm_add_affected_planes(state, crtc);
		if (ret)
			return ret;
	}

6438
	skl_print_wm_changes(state);
6439

6440 6441 6442
	return 0;
}

6443
static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6444 6445 6446 6447 6448
				  struct intel_wm_config *config)
{
	struct intel_crtc *crtc;

	/* Compute the currently _active_ config */
6449
	for_each_intel_crtc(&dev_priv->drm, crtc) {
6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460
		const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;

		if (!wm->pipe_enabled)
			continue;

		config->sprites_enabled |= wm->sprites_enabled;
		config->sprites_scaled |= wm->sprites_scaled;
		config->num_pipes_active++;
	}
}

6461
static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6462
{
6463
	struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6464
	struct ilk_wm_maximums max;
6465
	struct intel_wm_config config = {};
6466
	struct ilk_wm_values results = {};
6467
	enum intel_ddb_partitioning partitioning;
6468

6469
	ilk_compute_wm_config(dev_priv, &config);
6470

6471 6472
	ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
	ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6473 6474

	/* 5/6 split only in single pipe config on IVB+ */
6475
	if (DISPLAY_VER(dev_priv) >= 7 &&
6476
	    config.num_pipes_active == 1 && config.sprites_enabled) {
6477 6478
		ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
		ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6479

6480
		best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6481
	} else {
6482
		best_lp_wm = &lp_wm_1_2;
6483 6484
	}

6485
	partitioning = (best_lp_wm == &lp_wm_1_2) ?
6486
		       INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6487

6488
	ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6489

6490
	ilk_write_wm_values(dev_priv, &results);
6491 6492
}

6493
static void ilk_initial_watermarks(struct intel_atomic_state *state,
6494
				   struct intel_crtc *crtc)
6495
{
6496 6497 6498
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	const struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
6499

6500
	mutex_lock(&dev_priv->wm.wm_mutex);
6501
	crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6502 6503 6504
	ilk_program_watermarks(dev_priv);
	mutex_unlock(&dev_priv->wm.wm_mutex);
}
6505

6506
static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6507
				    struct intel_crtc *crtc)
6508
{
6509 6510 6511
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	const struct intel_crtc_state *crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);
6512 6513 6514

	if (!crtc_state->wm.need_postvbl_update)
		return;
6515

6516
	mutex_lock(&dev_priv->wm.wm_mutex);
6517 6518
	crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
	ilk_program_watermarks(dev_priv);
6519
	mutex_unlock(&dev_priv->wm.wm_mutex);
6520 6521
}

6522
static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6523
{
6524
	level->enable = val & PLANE_WM_EN;
6525
	level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6526
	level->blocks = val & PLANE_WM_BLOCKS_MASK;
6527
	level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
6528 6529
}

6530
void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6531
			      struct skl_pipe_wm *out)
6532
{
6533 6534
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
	enum pipe pipe = crtc->pipe;
6535 6536
	int level, max_level;
	enum plane_id plane_id;
6537
	u32 val;
6538

6539
	max_level = ilk_wm_max_level(dev_priv);
6540

6541
	for_each_plane_id_on_crtc(crtc, plane_id) {
6542
		struct skl_plane_wm *wm = &out->planes[plane_id];
6543

6544
		for (level = 0; level <= max_level; level++) {
6545
			if (plane_id != PLANE_CURSOR)
6546
				val = intel_uncore_read(&dev_priv->uncore, PLANE_WM(pipe, plane_id, level));
6547
			else
6548
				val = intel_uncore_read(&dev_priv->uncore, CUR_WM(pipe, level));
6549

6550
			skl_wm_level_from_reg_val(val, &wm->wm[level]);
6551 6552
		}

6553
		if (plane_id != PLANE_CURSOR)
6554
			val = intel_uncore_read(&dev_priv->uncore, PLANE_WM_TRANS(pipe, plane_id));
6555
		else
6556
			val = intel_uncore_read(&dev_priv->uncore, CUR_WM_TRANS(pipe));
6557 6558

		skl_wm_level_from_reg_val(val, &wm->trans_wm);
6559

6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573 6574 6575 6576 6577 6578
		if (HAS_HW_SAGV_WM(dev_priv)) {
			if (plane_id != PLANE_CURSOR)
				val = intel_uncore_read(&dev_priv->uncore,
							PLANE_WM_SAGV(pipe, plane_id));
			else
				val = intel_uncore_read(&dev_priv->uncore,
							CUR_WM_SAGV(pipe));

			skl_wm_level_from_reg_val(val, &wm->sagv.wm0);

			if (plane_id != PLANE_CURSOR)
				val = intel_uncore_read(&dev_priv->uncore,
							PLANE_WM_SAGV_TRANS(pipe, plane_id));
			else
				val = intel_uncore_read(&dev_priv->uncore,
							CUR_WM_SAGV_TRANS(pipe));

			skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
		} else if (DISPLAY_VER(dev_priv) >= 12) {
6579 6580 6581
			wm->sagv.wm0 = wm->wm[0];
			wm->sagv.trans_wm = wm->trans_wm;
		}
6582 6583 6584
	}
}

6585
void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6586
{
6587 6588
	struct intel_dbuf_state *dbuf_state =
		to_intel_dbuf_state(dev_priv->dbuf.obj.state);
6589
	struct intel_crtc *crtc;
6590

6591 6592 6593
	if (IS_ALDERLAKE_P(dev_priv))
		dbuf_state->joined_mbus = intel_de_read(dev_priv, MBUS_CTL) & MBUS_JOIN;

6594
	for_each_intel_crtc(&dev_priv->drm, crtc) {
6595 6596 6597
		struct intel_crtc_state *crtc_state =
			to_intel_crtc_state(crtc->base.state);
		enum pipe pipe = crtc->pipe;
6598
		unsigned int mbus_offset;
6599
		enum plane_id plane_id;
6600

6601
		skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6602
		crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622 6623

		memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));

		for_each_plane_id_on_crtc(crtc, plane_id) {
			struct skl_ddb_entry *ddb_y =
				&crtc_state->wm.skl.plane_ddb_y[plane_id];
			struct skl_ddb_entry *ddb_uv =
				&crtc_state->wm.skl.plane_ddb_uv[plane_id];

			skl_ddb_get_hw_plane_state(dev_priv, crtc->pipe,
						   plane_id, ddb_y, ddb_uv);

			skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
			skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_uv);
		}

		dbuf_state->slices[pipe] =
			skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes);

		dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);

6624 6625 6626 6627 6628 6629 6630
		/*
		 * Used for checking overlaps, so we need absolute
		 * offsets instead of MBUS relative offsets.
		 */
		mbus_offset = mbus_ddb_offset(dev_priv, dbuf_state->slices[pipe]);
		crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
		crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
6631 6632

		drm_dbg_kms(&dev_priv->drm,
6633
			    "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
6634 6635
			    crtc->base.base.id, crtc->base.name,
			    dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
6636 6637
			    dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
			    yesno(dbuf_state->joined_mbus));
6638
	}
6639 6640

	dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices;
6641 6642
}

6643
static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6644
{
6645
	struct drm_device *dev = crtc->base.dev;
6646
	struct drm_i915_private *dev_priv = to_i915(dev);
6647
	struct ilk_wm_values *hw = &dev_priv->wm.hw;
6648 6649
	struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
	struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6650
	enum pipe pipe = crtc->pipe;
6651

6652
	hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
6653

6654 6655
	memset(active, 0, sizeof(*active));

6656
	active->pipe_enabled = crtc->active;
6657 6658

	if (active->pipe_enabled) {
6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671
		u32 tmp = hw->wm_pipe[pipe];

		/*
		 * For active pipes LP0 watermark is marked as
		 * enabled, and LP1+ watermaks as disabled since
		 * we can't really reverse compute them in case
		 * multiple pipes are active.
		 */
		active->wm[0].enable = true;
		active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
		active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
		active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
	} else {
6672
		int level, max_level = ilk_wm_max_level(dev_priv);
6673 6674 6675 6676 6677 6678 6679 6680 6681

		/*
		 * For inactive pipes, all watermark levels
		 * should be marked as enabled but zeroed,
		 * which is what we'd compute them to.
		 */
		for (level = 0; level <= max_level; level++)
			active->wm[level].enable = true;
	}
6682

6683
	crtc->wm.active.ilk = *active;
6684 6685
}

6686 6687 6688 6689 6690
#define _FW_WM(value, plane) \
	(((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
#define _FW_WM_VLV(value, plane) \
	(((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)

6691 6692 6693
static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
			       struct g4x_wm_values *wm)
{
6694
	u32 tmp;
6695

6696
	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6697 6698 6699 6700 6701
	wm->sr.plane = _FW_WM(tmp, SR);
	wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);

6702
	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6703 6704 6705 6706 6707 6708 6709
	wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
	wm->sr.fbc = _FW_WM(tmp, FBC_SR);
	wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
	wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
	wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);

6710
	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6711 6712 6713 6714 6715 6716
	wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
	wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
	wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
}

6717 6718 6719 6720
static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
			       struct vlv_wm_values *wm)
{
	enum pipe pipe;
6721
	u32 tmp;
6722 6723

	for_each_pipe(dev_priv, pipe) {
6724
		tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
6725

6726
		wm->ddl[pipe].plane[PLANE_PRIMARY] =
6727
			(tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6728
		wm->ddl[pipe].plane[PLANE_CURSOR] =
6729
			(tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6730
		wm->ddl[pipe].plane[PLANE_SPRITE0] =
6731
			(tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6732
		wm->ddl[pipe].plane[PLANE_SPRITE1] =
6733 6734 6735
			(tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
	}

6736
	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6737
	wm->sr.plane = _FW_WM(tmp, SR);
6738 6739 6740
	wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6741

6742
	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6743 6744 6745
	wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
	wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6746

6747
	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6748 6749 6750
	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);

	if (IS_CHERRYVIEW(dev_priv)) {
6751
		tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
6752 6753
		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6754

6755
		tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
6756 6757
		wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
		wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6758

6759
		tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
6760 6761
		wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
		wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6762

6763
		tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6764
		wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6765 6766 6767 6768 6769 6770 6771 6772 6773
		wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
		wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
		wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
		wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
		wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
		wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
		wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6774
	} else {
6775
		tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
6776 6777
		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6778

6779
		tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6780
		wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6781 6782 6783 6784 6785 6786
		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
		wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
		wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
		wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
		wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6787 6788 6789 6790 6791 6792
	}
}

#undef _FW_WM
#undef _FW_WM_VLV

6793
void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6794 6795 6796 6797 6798 6799
{
	struct g4x_wm_values *wm = &dev_priv->wm.g4x;
	struct intel_crtc *crtc;

	g4x_read_wm_values(dev_priv, wm);

6800
	wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
6801

6802
	for_each_intel_crtc(&dev_priv->drm, crtc) {
6803 6804 6805 6806 6807 6808 6809 6810 6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844 6845 6846 6847 6848 6849 6850 6851 6852 6853 6854 6855 6856 6857 6858 6859 6860 6861
		struct intel_crtc_state *crtc_state =
			to_intel_crtc_state(crtc->base.state);
		struct g4x_wm_state *active = &crtc->wm.active.g4x;
		struct g4x_pipe_wm *raw;
		enum pipe pipe = crtc->pipe;
		enum plane_id plane_id;
		int level, max_level;

		active->cxsr = wm->cxsr;
		active->hpll_en = wm->hpll_en;
		active->fbc_en = wm->fbc_en;

		active->sr = wm->sr;
		active->hpll = wm->hpll;

		for_each_plane_id_on_crtc(crtc, plane_id) {
			active->wm.plane[plane_id] =
				wm->pipe[pipe].plane[plane_id];
		}

		if (wm->cxsr && wm->hpll_en)
			max_level = G4X_WM_LEVEL_HPLL;
		else if (wm->cxsr)
			max_level = G4X_WM_LEVEL_SR;
		else
			max_level = G4X_WM_LEVEL_NORMAL;

		level = G4X_WM_LEVEL_NORMAL;
		raw = &crtc_state->wm.g4x.raw[level];
		for_each_plane_id_on_crtc(crtc, plane_id)
			raw->plane[plane_id] = active->wm.plane[plane_id];

		if (++level > max_level)
			goto out;

		raw = &crtc_state->wm.g4x.raw[level];
		raw->plane[PLANE_PRIMARY] = active->sr.plane;
		raw->plane[PLANE_CURSOR] = active->sr.cursor;
		raw->plane[PLANE_SPRITE0] = 0;
		raw->fbc = active->sr.fbc;

		if (++level > max_level)
			goto out;

		raw = &crtc_state->wm.g4x.raw[level];
		raw->plane[PLANE_PRIMARY] = active->hpll.plane;
		raw->plane[PLANE_CURSOR] = active->hpll.cursor;
		raw->plane[PLANE_SPRITE0] = 0;
		raw->fbc = active->hpll.fbc;

	out:
		for_each_plane_id_on_crtc(crtc, plane_id)
			g4x_raw_plane_wm_set(crtc_state, level,
					     plane_id, USHRT_MAX);
		g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);

		crtc_state->wm.g4x.optimal = *active;
		crtc_state->wm.g4x.intermediate = *active;

6862 6863 6864 6865 6866 6867
		drm_dbg_kms(&dev_priv->drm,
			    "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
			    pipe_name(pipe),
			    wm->pipe[pipe].plane[PLANE_PRIMARY],
			    wm->pipe[pipe].plane[PLANE_CURSOR],
			    wm->pipe[pipe].plane[PLANE_SPRITE0]);
6868 6869
	}

6870 6871 6872 6873 6874 6875 6876 6877
	drm_dbg_kms(&dev_priv->drm,
		    "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
		    wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
	drm_dbg_kms(&dev_priv->drm,
		    "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
		    wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
	drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
		    yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897
}

void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
{
	struct intel_plane *plane;
	struct intel_crtc *crtc;

	mutex_lock(&dev_priv->wm.wm_mutex);

	for_each_intel_plane(&dev_priv->drm, plane) {
		struct intel_crtc *crtc =
			intel_get_crtc_for_pipe(dev_priv, plane->pipe);
		struct intel_crtc_state *crtc_state =
			to_intel_crtc_state(crtc->base.state);
		struct intel_plane_state *plane_state =
			to_intel_plane_state(plane->base.state);
		struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
		enum plane_id plane_id = plane->id;
		int level;

6898
		if (plane_state->uapi.visible)
6899 6900 6901 6902 6903 6904 6905 6906 6907 6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935
			continue;

		for (level = 0; level < 3; level++) {
			struct g4x_pipe_wm *raw =
				&crtc_state->wm.g4x.raw[level];

			raw->plane[plane_id] = 0;
			wm_state->wm.plane[plane_id] = 0;
		}

		if (plane_id == PLANE_PRIMARY) {
			for (level = 0; level < 3; level++) {
				struct g4x_pipe_wm *raw =
					&crtc_state->wm.g4x.raw[level];
				raw->fbc = 0;
			}

			wm_state->sr.fbc = 0;
			wm_state->hpll.fbc = 0;
			wm_state->fbc_en = false;
		}
	}

	for_each_intel_crtc(&dev_priv->drm, crtc) {
		struct intel_crtc_state *crtc_state =
			to_intel_crtc_state(crtc->base.state);

		crtc_state->wm.g4x.intermediate =
			crtc_state->wm.g4x.optimal;
		crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
	}

	g4x_program_watermarks(dev_priv);

	mutex_unlock(&dev_priv->wm.wm_mutex);
}

6936
void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6937 6938
{
	struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6939
	struct intel_crtc *crtc;
6940 6941 6942 6943
	u32 val;

	vlv_read_wm_values(dev_priv, wm);

6944
	wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6945 6946 6947
	wm->level = VLV_WM_LEVEL_PM2;

	if (IS_CHERRYVIEW(dev_priv)) {
6948
		vlv_punit_get(dev_priv);
6949

6950
		val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6951 6952 6953
		if (val & DSP_MAXFIFO_PM5_ENABLE)
			wm->level = VLV_WM_LEVEL_PM5;

6954 6955 6956 6957 6958 6959 6960 6961 6962
		/*
		 * If DDR DVFS is disabled in the BIOS, Punit
		 * will never ack the request. So if that happens
		 * assume we don't have to enable/disable DDR DVFS
		 * dynamically. To test that just set the REQ_ACK
		 * bit to poke the Punit, but don't change the
		 * HIGH/LOW bits so that we don't actually change
		 * the current state.
		 */
6963
		val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6964 6965 6966 6967 6968
		val |= FORCE_DDR_FREQ_REQ_ACK;
		vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);

		if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
			      FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6969 6970 6971
			drm_dbg_kms(&dev_priv->drm,
				    "Punit not acking DDR DVFS request, "
				    "assuming DDR DVFS is disabled\n");
6972 6973 6974 6975 6976 6977
			dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
		} else {
			val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
			if ((val & FORCE_DDR_HIGH_FREQ) == 0)
				wm->level = VLV_WM_LEVEL_DDR_DVFS;
		}
6978

6979
		vlv_punit_put(dev_priv);
6980 6981
	}

6982
	for_each_intel_crtc(&dev_priv->drm, crtc) {
6983 6984 6985 6986 6987 6988 6989 6990 6991 6992 6993 6994 6995 6996 6997
		struct intel_crtc_state *crtc_state =
			to_intel_crtc_state(crtc->base.state);
		struct vlv_wm_state *active = &crtc->wm.active.vlv;
		const struct vlv_fifo_state *fifo_state =
			&crtc_state->wm.vlv.fifo_state;
		enum pipe pipe = crtc->pipe;
		enum plane_id plane_id;
		int level;

		vlv_get_fifo_size(crtc_state);

		active->num_levels = wm->level + 1;
		active->cxsr = wm->cxsr;

		for (level = 0; level < active->num_levels; level++) {
6998
			struct g4x_pipe_wm *raw =
6999 7000 7001 7002 7003 7004 7005 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7017 7018 7019
				&crtc_state->wm.vlv.raw[level];

			active->sr[level].plane = wm->sr.plane;
			active->sr[level].cursor = wm->sr.cursor;

			for_each_plane_id_on_crtc(crtc, plane_id) {
				active->wm[level].plane[plane_id] =
					wm->pipe[pipe].plane[plane_id];

				raw->plane[plane_id] =
					vlv_invert_wm_value(active->wm[level].plane[plane_id],
							    fifo_state->plane[plane_id]);
			}
		}

		for_each_plane_id_on_crtc(crtc, plane_id)
			vlv_raw_plane_wm_set(crtc_state, level,
					     plane_id, USHRT_MAX);
		vlv_invalidate_wms(crtc, active, level);

		crtc_state->wm.vlv.optimal = *active;
7020
		crtc_state->wm.vlv.intermediate = *active;
7021

7022 7023 7024 7025 7026 7027 7028
		drm_dbg_kms(&dev_priv->drm,
			    "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
			    pipe_name(pipe),
			    wm->pipe[pipe].plane[PLANE_PRIMARY],
			    wm->pipe[pipe].plane[PLANE_CURSOR],
			    wm->pipe[pipe].plane[PLANE_SPRITE0],
			    wm->pipe[pipe].plane[PLANE_SPRITE1]);
7029
	}
7030

7031 7032 7033
	drm_dbg_kms(&dev_priv->drm,
		    "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
		    wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
7034 7035
}

7036 7037 7038 7039 7040 7041 7042 7043 7044 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054 7055
void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
{
	struct intel_plane *plane;
	struct intel_crtc *crtc;

	mutex_lock(&dev_priv->wm.wm_mutex);

	for_each_intel_plane(&dev_priv->drm, plane) {
		struct intel_crtc *crtc =
			intel_get_crtc_for_pipe(dev_priv, plane->pipe);
		struct intel_crtc_state *crtc_state =
			to_intel_crtc_state(crtc->base.state);
		struct intel_plane_state *plane_state =
			to_intel_plane_state(plane->base.state);
		struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
		const struct vlv_fifo_state *fifo_state =
			&crtc_state->wm.vlv.fifo_state;
		enum plane_id plane_id = plane->id;
		int level;

7056
		if (plane_state->uapi.visible)
7057 7058 7059
			continue;

		for (level = 0; level < wm_state->num_levels; level++) {
7060
			struct g4x_pipe_wm *raw =
7061 7062 7063 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084
				&crtc_state->wm.vlv.raw[level];

			raw->plane[plane_id] = 0;

			wm_state->wm[level].plane[plane_id] =
				vlv_invert_wm_value(raw->plane[plane_id],
						    fifo_state->plane[plane_id]);
		}
	}

	for_each_intel_crtc(&dev_priv->drm, crtc) {
		struct intel_crtc_state *crtc_state =
			to_intel_crtc_state(crtc->base.state);

		crtc_state->wm.vlv.intermediate =
			crtc_state->wm.vlv.optimal;
		crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
	}

	vlv_program_watermarks(dev_priv);

	mutex_unlock(&dev_priv->wm.wm_mutex);
}

7085 7086 7087 7088 7089 7090
/*
 * FIXME should probably kill this and improve
 * the real watermark readout/sanitation instead
 */
static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
{
7091 7092 7093
	intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM1_LP_SR_EN);
	intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM1_LP_SR_EN);
	intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM1_LP_SR_EN);
7094 7095 7096 7097 7098 7099 7100

	/*
	 * Don't touch WM1S_LP_EN here.
	 * Doing so could cause underruns.
	 */
}

7101
void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
7102
{
7103
	struct ilk_wm_values *hw = &dev_priv->wm.hw;
7104
	struct intel_crtc *crtc;
7105

7106 7107
	ilk_init_lp_watermarks(dev_priv);

7108
	for_each_intel_crtc(&dev_priv->drm, crtc)
7109 7110
		ilk_pipe_wm_get_hw_state(crtc);

7111 7112 7113
	hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
	hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
	hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
7114

7115
	hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
7116
	if (DISPLAY_VER(dev_priv) >= 7) {
7117 7118
		hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
		hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
7119
	}
7120

7121
	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
7122
		hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
7123
			INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7124
	else if (IS_IVYBRIDGE(dev_priv))
7125
		hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
7126
			INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7127 7128

	hw->enable_fbc_wm =
7129
		!(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
7130 7131
}

7132 7133
/**
 * intel_update_watermarks - update FIFO watermark values based on current modes
7134
 * @crtc: the #intel_crtc on which to compute the WM
7135 7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7160 7161 7162 7163 7164
 *
 * Calculate watermark values for the various WM regs based on current mode
 * and plane configuration.
 *
 * There are several cases to deal with here:
 *   - normal (i.e. non-self-refresh)
 *   - self-refresh (SR) mode
 *   - lines are large relative to FIFO size (buffer can hold up to 2)
 *   - lines are small relative to FIFO size (buffer can hold more than 2
 *     lines), so need to account for TLB latency
 *
 *   The normal calculation is:
 *     watermark = dotclock * bytes per pixel * latency
 *   where latency is platform & configuration dependent (we assume pessimal
 *   values here).
 *
 *   The SR calculation is:
 *     watermark = (trunc(latency/line time)+1) * surface width *
 *       bytes per pixel
 *   where
 *     line time = htotal / dotclock
 *     surface width = hdisplay for normal plane and 64 for cursor
 *   and latency is assumed to be high, as above.
 *
 * The final value programmed to the register should always be rounded up,
 * and include an extra 2 entries to account for clock crossings.
 *
 * We don't use the sprite, so we can ignore that.  And on Crestline we have
 * to set the non-SR watermarks to 8.
 */
7165
void intel_update_watermarks(struct intel_crtc *crtc)
7166
{
7167
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7168 7169

	if (dev_priv->display.update_wm)
7170
		dev_priv->display.update_wm(crtc);
7171 7172
}

7173 7174 7175 7176
void intel_enable_ipc(struct drm_i915_private *dev_priv)
{
	u32 val;

7177 7178 7179
	if (!HAS_IPC(dev_priv))
		return;

7180
	val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
7181 7182 7183 7184 7185 7186

	if (dev_priv->ipc_enabled)
		val |= DISP_IPC_ENABLE;
	else
		val &= ~DISP_IPC_ENABLE;

7187
	intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
7188 7189
}

7190 7191 7192 7193 7194 7195 7196
static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
{
	/* Display WA #0477 WaDisableIPC: skl */
	if (IS_SKYLAKE(dev_priv))
		return false;

	/* Display WA #1141: SKL:all KBL:all CFL */
7197 7198 7199
	if (IS_KABYLAKE(dev_priv) ||
	    IS_COFFEELAKE(dev_priv) ||
	    IS_COMETLAKE(dev_priv))
7200 7201 7202 7203 7204
		return dev_priv->dram_info.symmetric_memory;

	return true;
}

7205 7206 7207 7208 7209
void intel_init_ipc(struct drm_i915_private *dev_priv)
{
	if (!HAS_IPC(dev_priv))
		return;

7210
	dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
7211

7212 7213 7214
	intel_enable_ipc(dev_priv);
}

7215 7216 7217 7218 7219 7220 7221
static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
{
	/*
	 * On Ibex Peak and Cougar Point, we need to disable clock
	 * gating for the panel power sequencer or it will fail to
	 * start up when no ports are active.
	 */
7222
	intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
7223
}
7224

7225
static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
7226
{
7227
	enum pipe pipe;
7228

7229
	for_each_pipe(dev_priv, pipe) {
7230 7231
		intel_uncore_write(&dev_priv->uncore, DSPCNTR(pipe),
			   intel_uncore_read(&dev_priv->uncore, DSPCNTR(pipe)) |
7232
			   DISPPLANE_TRICKLE_FEED_DISABLE);
7233

7234 7235
		intel_uncore_write(&dev_priv->uncore, DSPSURF(pipe), intel_uncore_read(&dev_priv->uncore, DSPSURF(pipe)));
		intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
7236 7237 7238
	}
}

7239
static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
7240
{
7241
	u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7242

7243 7244 7245 7246 7247 7248 7249
	/*
	 * Required for FBC
	 * WaFbcDisableDpfcClockGating:ilk
	 */
	dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
		   ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
		   ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
7250

7251
	intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
7252 7253
		   MARIUNIT_CLOCK_GATE_DISABLE |
		   SVSMUNIT_CLOCK_GATE_DISABLE);
7254
	intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
7255
		   VFMUNIT_CLOCK_GATE_DISABLE);
7256

7257 7258 7259 7260 7261 7262 7263
	/*
	 * According to the spec the following bits should be set in
	 * order to enable memory self-refresh
	 * The bit 22/21 of 0x42004
	 * The bit 5 of 0x42020
	 * The bit 15 of 0x45000
	 */
7264 7265
	intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
		   (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7266 7267
		    ILK_DPARB_GATE | ILK_VSDPFD_FULL));
	dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
7268 7269
	intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
		   (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7270
		    DISP_FBC_WM_DIS));
7271 7272

	/*
7273 7274 7275 7276 7277
	 * Based on the document from hardware guys the following bits
	 * should be set unconditionally in order to enable FBC.
	 * The bit 22 of 0x42000
	 * The bit 22 of 0x42004
	 * The bit 7,8,9 of 0x42020.
7278
	 */
7279 7280
	if (IS_IRONLAKE_M(dev_priv)) {
		/* WaFbcAsynchFlipDisableFbcQueue:ilk */
7281 7282
		intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
			   intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7283
			   ILK_FBCQ_DIS);
7284 7285
		intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
			   intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7286 7287
			   ILK_DPARB_GATE);
	}
7288

7289
	intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7290

7291 7292
	intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
		   intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7293
		   ILK_ELPIN_409_SELECT);
7294

7295
	g4x_disable_trickle_feed(dev_priv);
7296

7297
	ibx_init_clock_gating(dev_priv);
7298 7299
}

7300
static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
7301
{
7302 7303
	enum pipe pipe;
	u32 val;
7304

7305 7306 7307 7308 7309
	/*
	 * On Ibex Peak and Cougar Point, we need to disable clock
	 * gating for the panel power sequencer or it will fail to
	 * start up when no ports are active.
	 */
7310
	intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
7311 7312
		   PCH_DPLUNIT_CLOCK_GATE_DISABLE |
		   PCH_CPUNIT_CLOCK_GATE_DISABLE);
7313
	intel_uncore_write(&dev_priv->uncore, SOUTH_CHICKEN2, intel_uncore_read(&dev_priv->uncore, SOUTH_CHICKEN2) |
7314 7315 7316 7317 7318
		   DPLS_EDP_PPS_FIX_DIS);
	/* The below fixes the weird display corruption, a few pixels shifted
	 * downward, on (only) LVDS of some HP laptops with IVY.
	 */
	for_each_pipe(dev_priv, pipe) {
7319
		val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
7320 7321 7322 7323 7324 7325
		val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
		val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
		if (dev_priv->vbt.fdi_rx_polarity_inverted)
			val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
		val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
		val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
7326
		intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
7327 7328 7329
	}
	/* WADP0ClockGatingDisable */
	for_each_pipe(dev_priv, pipe) {
7330
		intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
7331 7332
			   TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
	}
7333 7334
}

7335
static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
7336
{
7337
	u32 tmp;
7338

7339
	tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
7340
	if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
7341 7342 7343
		drm_dbg_kms(&dev_priv->drm,
			    "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
			    tmp);
7344 7345
}

7346
static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
7347
{
7348
	u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7349

7350
	intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7351

7352 7353
	intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
		   intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7354
		   ILK_ELPIN_409_SELECT);
7355

7356 7357
	intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
		   intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7358 7359
		   GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
		   GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7360

7361 7362 7363 7364 7365 7366 7367 7368 7369 7370 7371 7372
	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
	 * gating disable must be set.  Failure to set it results in
	 * flickering pixels due to Z write ordering failures after
	 * some amount of runtime in the Mesa "fire" demo, and Unigine
	 * Sanctuary and Tropics, and apparently anything else with
	 * alpha test or pixel discard.
	 *
	 * According to the spec, bit 11 (RCCUNIT) must also be set,
	 * but we didn't debug actual testcases to find it out.
	 *
	 * WaDisableRCCUnitClockGating:snb
	 * WaDisableRCPBUnitClockGating:snb
7373
	 */
7374
	intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7375 7376
		   GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
		   GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7377

7378 7379 7380 7381 7382 7383 7384 7385 7386 7387 7388
	/*
	 * According to the spec the following bits should be
	 * set in order to enable memory self-refresh and fbc:
	 * The bit21 and bit22 of 0x42000
	 * The bit21 and bit22 of 0x42004
	 * The bit5 and bit7 of 0x42020
	 * The bit14 of 0x70180
	 * The bit14 of 0x71180
	 *
	 * WaFbcAsynchFlipDisableFbcQueue:snb
	 */
7389 7390
	intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
		   intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7391
		   ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7392 7393
	intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
		   intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7394
		   ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7395 7396
	intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
		   intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
7397 7398
		   ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
		   ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7399

7400
	g4x_disable_trickle_feed(dev_priv);
7401

7402
	cpt_init_clock_gating(dev_priv);
7403

7404
	gen6_check_mch_setup(dev_priv);
7405 7406
}

7407
static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7408
{
7409 7410 7411
	/*
	 * TODO: this bit should only be enabled when really needed, then
	 * disabled when not needed anymore in order to save power.
7412
	 */
7413
	if (HAS_PCH_LPT_LP(dev_priv))
7414 7415
		intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
			   intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7416
			   PCH_LP_PARTITION_LEVEL_DISABLE);
7417 7418

	/* WADPOClockGatingDisable:hsw */
7419 7420
	intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
		   intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A)) |
7421
		   TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7422 7423
}

7424
static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7425
{
7426
	if (HAS_PCH_LPT_LP(dev_priv)) {
7427
		u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
7428 7429

		val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7430
		intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
7431 7432 7433
	}
}

7434 7435 7436 7437 7438
static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
				   int general_prio_credits,
				   int high_prio_credits)
{
	u32 misccpctl;
7439
	u32 val;
7440 7441

	/* WaTempDisableDOPClkGating:bdw */
7442 7443
	misccpctl = intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL);
	intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7444

7445
	val = intel_uncore_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7446 7447 7448
	val &= ~L3_PRIO_CREDITS_MASK;
	val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
	val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7449
	intel_uncore_write(&dev_priv->uncore, GEN8_L3SQCREG1, val);
7450 7451 7452 7453 7454

	/*
	 * Wait at least 100 clocks before re-enabling clock gating.
	 * See the definition of L3SQCREG1 in BSpec.
	 */
7455
	intel_uncore_posting_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7456
	udelay(1);
7457
	intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
7458 7459
}

7460 7461
static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
{
7462
	/* Wa_1409120013:icl,ehl */
7463
	intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN,
7464 7465
		   ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);

7466
	/* This is not an Wa. Enable to reduce Sampler power */
7467 7468
	intel_uncore_write(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
		   intel_uncore_read(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
7469

7470 7471
	/*Wa_14010594013:icl, ehl */
	intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7472
			 0, ICL_DELAY_PMRSP);
7473 7474
}

7475
static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
7476
{
7477
	/* Wa_1409120013:tgl,rkl,adl_s,dg1 */
7478
	intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN,
7479
			   ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7480

7481
	/* Wa_1409825376:tgl (pre-prod)*/
7482
	if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
7483
		intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7484
			   TGL_VRH_GATING_DIS);
Matt Atwood's avatar
Matt Atwood committed
7485

7486
	/* Wa_14011059788:tgl,rkl,adl_s,dg1 */
Matt Atwood's avatar
Matt Atwood committed
7487 7488
	intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
			 0, DFR_DISABLE);
7489 7490 7491 7492 7493

	/* Wa_14013723622:tgl,rkl,dg1,adl-s */
	if (DISPLAY_VER(dev_priv) == 12)
		intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
				 CLKREQ_POLICY_MEM_UP_OVRD, 0);
7494 7495
}

7496 7497 7498 7499 7500 7501 7502 7503
static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
{
	gen12lp_init_clock_gating(dev_priv);

	/* Wa_22011091694:adlp */
	intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
}

7504 7505
static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
{
7506 7507
	gen12lp_init_clock_gating(dev_priv);

7508
	/* Wa_1409836686:dg1[a0] */
7509
	if (IS_DG1_GT_STEP(dev_priv, STEP_A0, STEP_A0))
7510
		intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7511 7512 7513
			   DPT_GATING_DIS);
}

7514 7515 7516 7517 7518
static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
{
	if (!HAS_PCH_CNP(dev_priv))
		return;

7519
	/* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7520
	intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7521
		   CNP_PWM_CGE_GATING_DISABLE);
7522 7523 7524 7525 7526 7527 7528
}

static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
{
	cnp_init_clock_gating(dev_priv);
	gen9_init_clock_gating(dev_priv);

7529
	/* WAC6entrylatency:cfl */
7530
	intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7531 7532
		   FBC_LLC_FULLY_OPEN);

7533 7534 7535 7536
	/*
	 * WaFbcTurnOffFbcWatermark:cfl
	 * Display WA #0562: cfl
	 */
7537
	intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7538 7539
		   DISP_FBC_WM_DIS);

7540 7541 7542 7543
	/*
	 * WaFbcNukeOnHostModify:cfl
	 * Display WA #0873: cfl
	 */
7544
	intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7545 7546 7547
		   ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
}

7548
static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7549
{
7550
	gen9_init_clock_gating(dev_priv);
7551

7552
	/* WAC6entrylatency:kbl */
7553
	intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7554 7555
		   FBC_LLC_FULLY_OPEN);

7556
	/* WaDisableSDEUnitClockGating:kbl */
7557
	if (IS_KBL_GT_STEP(dev_priv, 0, STEP_B0))
7558
		intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7559
			   GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7560 7561

	/* WaDisableGamClockGating:kbl */
7562
	if (IS_KBL_GT_STEP(dev_priv, 0, STEP_B0))
7563
		intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7564
			   GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7565

7566 7567 7568 7569
	/*
	 * WaFbcTurnOffFbcWatermark:kbl
	 * Display WA #0562: kbl
	 */
7570
	intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7571 7572
		   DISP_FBC_WM_DIS);

7573 7574 7575 7576
	/*
	 * WaFbcNukeOnHostModify:kbl
	 * Display WA #0873: kbl
	 */
7577
	intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7578
		   ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7579 7580
}

7581
static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7582
{
7583
	gen9_init_clock_gating(dev_priv);
7584

7585
	/* WaDisableDopClockGating:skl */
7586
	intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL) &
7587 7588
		   ~GEN7_DOP_CLOCK_GATE_ENABLE);

7589
	/* WAC6entrylatency:skl */
7590
	intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7591
		   FBC_LLC_FULLY_OPEN);
7592

7593 7594 7595 7596
	/*
	 * WaFbcTurnOffFbcWatermark:skl
	 * Display WA #0562: skl
	 */
7597
	intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7598 7599
		   DISP_FBC_WM_DIS);

7600 7601 7602 7603
	/*
	 * WaFbcNukeOnHostModify:skl
	 * Display WA #0873: skl
	 */
7604
	intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7605
		   ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7606

7607 7608 7609 7610
	/*
	 * WaFbcHighMemBwCorruptionAvoidance:skl
	 * Display WA #0883: skl
	 */
7611
	intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7612
		   ILK_DPFC_DISABLE_DUMMY0);
7613 7614
}

7615
static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7616
{
7617
	enum pipe pipe;
7618

7619
	/* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7620 7621
	intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
		   intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7622 7623
		   HSW_FBCQ_DIS);

7624
	/* WaSwitchSolVfFArbitrationPriority:bdw */
7625
	intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7626

7627
	/* WaPsrDPAMaskVBlankInSRD:bdw */
7628 7629
	intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
		   intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7630

7631
	for_each_pipe(dev_priv, pipe) {
7632
		/* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7633 7634
		intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
			   intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe)) |
7635
			   BDW_DPRS_MASK_VBLANK_SRD);
7636 7637 7638 7639 7640

		/* Undocumented but fixes async flip + VT-d corruption */
		if (intel_vtd_active())
			intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
					 HSW_PRI_STRETCH_MAX_MASK, HSW_PRI_STRETCH_MAX_X1);
7641
	}
7642

7643 7644
	/* WaVSRefCountFullforceMissDisable:bdw */
	/* WaDSRefCountFullforceMissDisable:bdw */
7645 7646
	intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
		   intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7647
		   ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7648

7649
	intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL,
7650
		   _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7651 7652

	/* WaDisableSDEUnitClockGating:bdw */
7653
	intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7654
		   GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7655

7656 7657
	/* WaProgramL3SqcReg1Default:bdw */
	gen8_set_l3sqc_credits(dev_priv, 30, 2);
7658

7659
	/* WaKVMNotificationOnConfigChange:bdw */
7660
	intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR2_1, intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR2_1)
7661 7662
		   | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);

7663
	lpt_init_clock_gating(dev_priv);
7664 7665 7666 7667 7668 7669

	/* WaDisableDopClockGating:bdw
	 *
	 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
	 * clock gating.
	 */
7670 7671
	intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
		   intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7672 7673
}

7674
static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7675
{
7676 7677
	enum pipe pipe;

7678
	/* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7679 7680
	intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
		   intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7681 7682
		   HSW_FBCQ_DIS);

7683 7684 7685 7686 7687 7688 7689
	for_each_pipe(dev_priv, pipe) {
		/* Undocumented but fixes async flip + VT-d corruption */
		if (intel_vtd_active())
			intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
					 HSW_PRI_STRETCH_MAX_MASK, HSW_PRI_STRETCH_MAX_X1);
	}

7690
	/* This is required by WaCatErrorRejectionIssue:hsw */
7691 7692
	intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		   intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7693
		   GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7694

7695
	/* WaSwitchSolVfFArbitrationPriority:hsw */
7696
	intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7697

7698
	lpt_init_clock_gating(dev_priv);
7699 7700
}

7701
static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7702
{
7703
	u32 snpcr;
7704

7705
	intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7706

7707
	/* WaFbcAsynchFlipDisableFbcQueue:ivb */
7708 7709
	intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
		   intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7710 7711
		   ILK_FBCQ_DIS);

7712
	/* WaDisableBackToBackFlipFix:ivb */
7713
	intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7714 7715 7716
		   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
		   CHICKEN3_DGMG_DONE_FIX_DISABLE);

7717
	if (IS_IVB_GT1(dev_priv))
7718
		intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7719
			   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7720 7721
	else {
		/* must write both registers */
7722
		intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7723
			   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7724
		intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
7725
			   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7726
	}
7727

7728
	/*
7729
	 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7730
	 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7731
	 */
7732
	intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7733
		   GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7734

7735
	/* This is required by WaCatErrorRejectionIssue:ivb */
7736 7737
	intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
			intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7738 7739
			GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);

7740
	g4x_disable_trickle_feed(dev_priv);
7741

7742
	snpcr = intel_uncore_read(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR);
7743 7744
	snpcr &= ~GEN6_MBC_SNPCR_MASK;
	snpcr |= GEN6_MBC_SNPCR_MED;
7745
	intel_uncore_write(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, snpcr);
7746

7747
	if (!HAS_PCH_NOP(dev_priv))
7748
		cpt_init_clock_gating(dev_priv);
7749

7750
	gen6_check_mch_setup(dev_priv);
7751 7752
}

7753
static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7754
{
7755
	/* WaDisableBackToBackFlipFix:vlv */
7756
	intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7757 7758 7759
		   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
		   CHICKEN3_DGMG_DONE_FIX_DISABLE);

7760
	/* WaDisableDopClockGating:vlv */
7761
	intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7762 7763
		   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));

7764
	/* This is required by WaCatErrorRejectionIssue:vlv */
7765 7766
	intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		   intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7767 7768
		   GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);

7769
	/*
7770
	 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7771
	 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7772
	 */
7773
	intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7774
		   GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7775

7776 7777 7778
	/* WaDisableL3Bank2xClockGate:vlv
	 * Disabling L3 clock gating- MMIO 940c[25] = 1
	 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7779 7780
	intel_uncore_write(&dev_priv->uncore, GEN7_UCGCTL4,
		   intel_uncore_read(&dev_priv->uncore, GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7781

7782
	/*
7783
	 * WaDisableVLVClockGating_VBIIssue:vlv
7784 7785 7786
	 * Disable clock gating on th GCFG unit to prevent a delay
	 * in the reporting of vblank events.
	 */
7787
	intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7788 7789
}

7790
static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7791
{
7792 7793
	/* WaVSRefCountFullforceMissDisable:chv */
	/* WaDSRefCountFullforceMissDisable:chv */
7794 7795
	intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
		   intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7796
		   ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7797 7798

	/* WaDisableSemaphoreAndSyncFlipWait:chv */
7799
	intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL,
7800
		   _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7801 7802

	/* WaDisableCSUnitClockGating:chv */
7803
	intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7804
		   GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7805 7806

	/* WaDisableSDEUnitClockGating:chv */
7807
	intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7808
		   GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7809

7810 7811 7812 7813 7814 7815
	/*
	 * WaProgramL3SqcReg1Default:chv
	 * See gfxspecs/Related Documents/Performance Guide/
	 * LSQC Setting Recommendations.
	 */
	gen8_set_l3sqc_credits(dev_priv, 38, 2);
7816 7817
}

7818
static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7819
{
7820
	u32 dspclk_gate;
7821

7822 7823
	intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
	intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7824 7825
		   GS_UNIT_CLOCK_GATE_DISABLE |
		   CL_UNIT_CLOCK_GATE_DISABLE);
7826
	intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
7827 7828 7829
	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
		OVRUNIT_CLOCK_GATE_DISABLE |
		OVCUNIT_CLOCK_GATE_DISABLE;
7830
	if (IS_GM45(dev_priv))
7831
		dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7832
	intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D, dspclk_gate);
7833

7834
	g4x_disable_trickle_feed(dev_priv);
7835 7836
}

7837
static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7838
{
7839 7840 7841 7842 7843 7844 7845 7846 7847 7848
	struct intel_uncore *uncore = &dev_priv->uncore;

	intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
	intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
	intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
	intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
	intel_uncore_write16(uncore, DEUC, 0);
	intel_uncore_write(uncore,
			   MI_ARB_STATE,
			   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7849 7850
}

7851
static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7852
{
7853
	intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7854 7855 7856 7857
		   I965_RCC_CLOCK_GATE_DISABLE |
		   I965_RCPB_CLOCK_GATE_DISABLE |
		   I965_ISC_CLOCK_GATE_DISABLE |
		   I965_FBC_CLOCK_GATE_DISABLE);
7858 7859
	intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
	intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7860
		   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7861 7862
}

7863
static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7864
{
7865
	u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
7866 7867 7868

	dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
		DSTATE_DOT_CLOCK_GATING;
7869
	intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
7870

7871
	if (IS_PINEVIEW(dev_priv))
7872
		intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7873 7874

	/* IIR "flip pending" means done if this bit is set */
7875
	intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7876 7877

	/* interrupts should cause a wake up from C3 */
7878
	intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7879 7880

	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7881
	intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7882

7883
	intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7884
		   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7885 7886
}

7887
static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7888
{
7889
	intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7890 7891

	/* interrupts should cause a wake up from C3 */
7892
	intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7893
		   _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7894

7895
	intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7896
		   _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7897 7898 7899 7900 7901 7902 7903 7904

	/*
	 * Have FBC ignore 3D activity since we use software
	 * render tracking, and otherwise a pure 3D workload
	 * (even if it just renders a single frame and then does
	 * abosultely nothing) would not allow FBC to recompress
	 * until a 2D blit occurs.
	 */
7905
	intel_uncore_write(&dev_priv->uncore, SCPD0,
7906
		   _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
7907 7908
}

7909
static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7910
{
7911
	intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7912 7913
		   _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
		   _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7914 7915
}

7916
void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7917
{
7918
	dev_priv->display.init_clock_gating(dev_priv);
7919 7920
}

7921
void intel_suspend_hw(struct drm_i915_private *dev_priv)
7922
{
7923 7924
	if (HAS_PCH_LPT(dev_priv))
		lpt_suspend_hw(dev_priv);
7925 7926
}

7927
static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7928
{
7929 7930
	drm_dbg_kms(&dev_priv->drm,
		    "No clock gating settings or workarounds applied.\n");
7931 7932 7933 7934 7935 7936 7937 7938 7939 7940 7941 7942 7943
}

/**
 * intel_init_clock_gating_hooks - setup the clock gating hooks
 * @dev_priv: device private
 *
 * Setup the hooks that configure which clocks of a given platform can be
 * gated and also apply various GT and display specific workarounds for these
 * platforms. Note that some GT specific workarounds are applied separately
 * when GPU contexts or batchbuffers start their execution.
 */
void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
{
7944 7945 7946
	if (IS_ALDERLAKE_P(dev_priv))
		dev_priv->display.init_clock_gating = adlp_init_clock_gating;
	else if (IS_DG1(dev_priv))
7947
		dev_priv->display.init_clock_gating = dg1_init_clock_gating;
7948
	else if (GRAPHICS_VER(dev_priv) == 12)
7949
		dev_priv->display.init_clock_gating = gen12lp_init_clock_gating;
7950
	else if (GRAPHICS_VER(dev_priv) == 11)
7951
		dev_priv->display.init_clock_gating = icl_init_clock_gating;
7952
	else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
7953
		dev_priv->display.init_clock_gating = cfl_init_clock_gating;
7954
	else if (IS_SKYLAKE(dev_priv))
7955
		dev_priv->display.init_clock_gating = skl_init_clock_gating;
7956
	else if (IS_KABYLAKE(dev_priv))
7957
		dev_priv->display.init_clock_gating = kbl_init_clock_gating;
7958
	else if (IS_BROXTON(dev_priv))
7959
		dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7960 7961
	else if (IS_GEMINILAKE(dev_priv))
		dev_priv->display.init_clock_gating = glk_init_clock_gating;
7962
	else if (IS_BROADWELL(dev_priv))
7963
		dev_priv->display.init_clock_gating = bdw_init_clock_gating;
7964
	else if (IS_CHERRYVIEW(dev_priv))
7965
		dev_priv->display.init_clock_gating = chv_init_clock_gating;
7966
	else if (IS_HASWELL(dev_priv))
7967
		dev_priv->display.init_clock_gating = hsw_init_clock_gating;
7968
	else if (IS_IVYBRIDGE(dev_priv))
7969
		dev_priv->display.init_clock_gating = ivb_init_clock_gating;
7970
	else if (IS_VALLEYVIEW(dev_priv))
7971
		dev_priv->display.init_clock_gating = vlv_init_clock_gating;
7972
	else if (GRAPHICS_VER(dev_priv) == 6)
7973
		dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7974
	else if (GRAPHICS_VER(dev_priv) == 5)
7975
		dev_priv->display.init_clock_gating = ilk_init_clock_gating;
7976 7977
	else if (IS_G4X(dev_priv))
		dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7978
	else if (IS_I965GM(dev_priv))
7979
		dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
7980
	else if (IS_I965G(dev_priv))
7981
		dev_priv->display.init_clock_gating = i965g_init_clock_gating;
7982
	else if (GRAPHICS_VER(dev_priv) == 3)
7983 7984 7985
		dev_priv->display.init_clock_gating = gen3_init_clock_gating;
	else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
		dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7986
	else if (GRAPHICS_VER(dev_priv) == 2)
7987 7988 7989 7990 7991 7992 7993
		dev_priv->display.init_clock_gating = i830_init_clock_gating;
	else {
		MISSING_CASE(INTEL_DEVID(dev_priv));
		dev_priv->display.init_clock_gating = nop_init_clock_gating;
	}
}

7994
/* Set up chip specific power management-related functions */
7995
void intel_init_pm(struct drm_i915_private *dev_priv)
7996
{
7997
	/* For cxsr */
7998
	if (IS_PINEVIEW(dev_priv))
7999
		pnv_get_mem_freq(dev_priv);
8000
	else if (GRAPHICS_VER(dev_priv) == 5)
8001
		ilk_get_mem_freq(dev_priv);
8002

8003 8004 8005
	if (intel_has_sagv(dev_priv))
		skl_setup_sagv_block_time(dev_priv);

8006
	/* For FIFO watermark updates */
8007
	if (DISPLAY_VER(dev_priv) >= 9) {
8008
		skl_setup_wm_latency(dev_priv);
8009
		dev_priv->display.compute_global_watermarks = skl_compute_wm;
8010
	} else if (HAS_PCH_SPLIT(dev_priv)) {
8011
		ilk_setup_wm_latency(dev_priv);
8012

8013
		if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->wm.pri_latency[1] &&
8014
		     dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
8015
		    (DISPLAY_VER(dev_priv) != 5 && dev_priv->wm.pri_latency[0] &&
8016
		     dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
8017
			dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
8018 8019 8020 8021 8022 8023
			dev_priv->display.compute_intermediate_wm =
				ilk_compute_intermediate_wm;
			dev_priv->display.initial_watermarks =
				ilk_initial_watermarks;
			dev_priv->display.optimize_watermarks =
				ilk_optimize_watermarks;
8024
		} else {
8025 8026 8027
			drm_dbg_kms(&dev_priv->drm,
				    "Failed to read display plane latency. "
				    "Disable CxSR\n");
8028
		}
8029
	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
8030
		vlv_setup_wm_latency(dev_priv);
8031
		dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
8032
		dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
8033
		dev_priv->display.initial_watermarks = vlv_initial_watermarks;
8034
		dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
8035
		dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
8036 8037 8038 8039 8040 8041
	} else if (IS_G4X(dev_priv)) {
		g4x_setup_wm_latency(dev_priv);
		dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
		dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
		dev_priv->display.initial_watermarks = g4x_initial_watermarks;
		dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
8042
	} else if (IS_PINEVIEW(dev_priv)) {
8043
		if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
8044 8045 8046
					    dev_priv->is_ddr3,
					    dev_priv->fsb_freq,
					    dev_priv->mem_freq)) {
8047 8048
			drm_info(&dev_priv->drm,
				 "failed to find known CxSR latency "
8049 8050 8051 8052 8053
				 "(found ddr%s fsb freq %d, mem freq %d), "
				 "disabling CxSR\n",
				 (dev_priv->is_ddr3 == 1) ? "3" : "2",
				 dev_priv->fsb_freq, dev_priv->mem_freq);
			/* Disable CxSR and never update its watermark again */
8054
			intel_set_memory_cxsr(dev_priv, false);
8055 8056
			dev_priv->display.update_wm = NULL;
		} else
8057
			dev_priv->display.update_wm = pnv_update_wm;
8058
	} else if (DISPLAY_VER(dev_priv) == 4) {
8059
		dev_priv->display.update_wm = i965_update_wm;
8060
	} else if (DISPLAY_VER(dev_priv) == 3) {
8061 8062
		dev_priv->display.update_wm = i9xx_update_wm;
		dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
8063
	} else if (DISPLAY_VER(dev_priv) == 2) {
8064
		if (INTEL_NUM_PIPES(dev_priv) == 1) {
8065
			dev_priv->display.update_wm = i845_update_wm;
8066
			dev_priv->display.get_fifo_size = i845_get_fifo_size;
8067 8068
		} else {
			dev_priv->display.update_wm = i9xx_update_wm;
8069
			dev_priv->display.get_fifo_size = i830_get_fifo_size;
8070 8071
		}
	} else {
8072 8073
		drm_err(&dev_priv->drm,
			"unexpected fall-through in %s\n", __func__);
8074 8075 8076
	}
}

8077
void intel_pm_setup(struct drm_i915_private *dev_priv)
8078
{
8079 8080
	dev_priv->runtime_pm.suspended = false;
	atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
8081
}
8082 8083 8084 8085 8086 8087 8088 8089 8090 8091 8092 8093 8094 8095 8096 8097 8098 8099 8100 8101 8102 8103 8104 8105 8106 8107 8108 8109 8110 8111 8112 8113 8114 8115 8116 8117 8118 8119 8120 8121 8122 8123 8124 8125 8126 8127 8128 8129 8130

static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
{
	struct intel_dbuf_state *dbuf_state;

	dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
	if (!dbuf_state)
		return NULL;

	return &dbuf_state->base;
}

static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
				     struct intel_global_state *state)
{
	kfree(state);
}

static const struct intel_global_state_funcs intel_dbuf_funcs = {
	.atomic_duplicate_state = intel_dbuf_duplicate_state,
	.atomic_destroy_state = intel_dbuf_destroy_state,
};

struct intel_dbuf_state *
intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
{
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
	struct intel_global_state *dbuf_state;

	dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
	if (IS_ERR(dbuf_state))
		return ERR_CAST(dbuf_state);

	return to_intel_dbuf_state(dbuf_state);
}

int intel_dbuf_init(struct drm_i915_private *dev_priv)
{
	struct intel_dbuf_state *dbuf_state;

	dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
	if (!dbuf_state)
		return -ENOMEM;

	intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
				     &dbuf_state->base, &intel_dbuf_funcs);

	return 0;
}
8131

8132 8133 8134 8135 8136 8137 8138 8139 8140 8141 8142 8143 8144 8145 8146 8147 8148 8149 8150 8151 8152 8153 8154 8155 8156 8157 8158 8159 8160 8161 8162 8163 8164 8165 8166 8167 8168 8169 8170
/*
 * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
 * update the request state of all DBUS slices.
 */
static void update_mbus_pre_enable(struct intel_atomic_state *state)
{
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
	u32 mbus_ctl, dbuf_min_tracker_val;
	enum dbuf_slice slice;
	const struct intel_dbuf_state *dbuf_state =
		intel_atomic_get_new_dbuf_state(state);

	if (!IS_ALDERLAKE_P(dev_priv))
		return;

	/*
	 * TODO: Implement vblank synchronized MBUS joining changes.
	 * Must be properly coordinated with dbuf reprogramming.
	 */
	if (dbuf_state->joined_mbus) {
		mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
			MBUS_JOIN_PIPE_SELECT_NONE;
		dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
	} else {
		mbus_ctl = MBUS_HASHING_MODE_2x2 |
			MBUS_JOIN_PIPE_SELECT_NONE;
		dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
	}

	intel_de_rmw(dev_priv, MBUS_CTL,
		     MBUS_HASHING_MODE_MASK | MBUS_JOIN |
		     MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);

	for_each_dbuf_slice(dev_priv, slice)
		intel_de_rmw(dev_priv, DBUF_CTL_S(slice),
			     DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
			     dbuf_min_tracker_val);
}

8171 8172 8173 8174 8175 8176 8177 8178 8179
void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
{
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
	const struct intel_dbuf_state *new_dbuf_state =
		intel_atomic_get_new_dbuf_state(state);
	const struct intel_dbuf_state *old_dbuf_state =
		intel_atomic_get_old_dbuf_state(state);

	if (!new_dbuf_state ||
8180 8181
	    ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
	    && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8182 8183 8184 8185
		return;

	WARN_ON(!new_dbuf_state->base.changed);

8186
	update_mbus_pre_enable(state);
8187 8188 8189 8190 8191 8192 8193 8194 8195 8196 8197 8198 8199 8200
	gen9_dbuf_slices_update(dev_priv,
				old_dbuf_state->enabled_slices |
				new_dbuf_state->enabled_slices);
}

void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
{
	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
	const struct intel_dbuf_state *new_dbuf_state =
		intel_atomic_get_new_dbuf_state(state);
	const struct intel_dbuf_state *old_dbuf_state =
		intel_atomic_get_old_dbuf_state(state);

	if (!new_dbuf_state ||
8201 8202
	    ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
	    && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8203 8204 8205 8206 8207 8208 8209
		return;

	WARN_ON(!new_dbuf_state->base.changed);

	gen9_dbuf_slices_update(dev_priv,
				new_dbuf_state->enabled_slices);
}