memory: tegra: Support derated timings on Tegra210

Derated timings are used to ensure that the memory chips keep operating
correctly at high temperatures. This adds code to support polling of the
chip operating state when high temperatures are measured on the chip and
change the refresh mode accordingly. Under very high temperatures, the
driver will switch to the derated tables to ensure proper operation of
the memory chips.
Signed-off-by: Thierry Reding <treding@nvidia.com>

drivers/memory/tegra/tegra210-emc-cc-r21021.c

@@ -1070,6 +1070,9 @@ static void tegra210_emc_r21021_set_clock(struct tegra210_emc *emc, u32 clksrc)
 		emc_writel(emc, value, offset);
 	}
 
+	/* SW addition: do EMC refresh adjustment here. */
+	tegra210_emc_adjust_timing(emc, next);
+
 	if (dram_type == DRAM_TYPE_LPDDR4) {
 		value = (23 << EMC_MRW_MRW_MA_SHIFT) |
 			(next->run_clocks & EMC_MRW_MRW_OP_MASK);

drivers/memory/tegra/tegra210-emc-core.c

@@ -3,6 +3,7 @@
  * Copyright (c) 2015-2020, NVIDIA CORPORATION.  All rights reserved.
  */
 
+#include <linux/bitfield.h>
 #include <linux/clk.h>
 #include <linux/clk/tegra.h>
 #include <linux/clk-provider.h>
@@ -14,6 +15,7 @@
 #include <linux/of_platform.h>
 #include <linux/of_reserved_mem.h>
 #include <linux/slab.h>
+#include <linux/thermal.h>
 #include <soc/tegra/fuse.h>
 #include <soc/tegra/mc.h>
 
@@ -26,6 +28,7 @@
 	(0x7 << EMC_CLK_EMC_2X_CLK_SRC_SHIFT)
 #define EMC_CLK_SOURCE_PLLM_LJ				0x4
 #define EMC_CLK_SOURCE_PLLMB_LJ				0x5
+#define EMC_CLK_FORCE_CC_TRIGGER			BIT(27)
 #define EMC_CLK_MC_EMC_SAME_FREQ			BIT(16)
 #define EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT		0
 #define EMC_CLK_EMC_2X_CLK_DIVISOR_MASK			\
@@ -83,6 +86,8 @@
 #define REFRESH_SPEEDUP(value, speedup) \
 		(((value) & 0xffff0000) | ((value) & 0xffff) * (speedup))
 
+#define LPDDR2_MR4_SRR GENMASK(2, 0)
+
 static const struct tegra210_emc_sequence *tegra210_emc_sequences[] = {
 	&tegra210_emc_r21021,
 };
@@ -582,6 +587,137 @@ static void tegra210_emc_training_stop(struct tegra210_emc *emc)
 	del_timer(&emc->training);
 }
 
+static unsigned int tegra210_emc_get_temperature(struct tegra210_emc *emc)
+{
+	unsigned long flags;
+	u32 value, max = 0;
+	unsigned int i;
+
+	spin_lock_irqsave(&emc->lock, flags);
+
+	for (i = 0; i < emc->num_devices; i++) {
+		value = tegra210_emc_mrr_read(emc, i, 4);
+
+		if (value & BIT(7))
+			dev_dbg(emc->dev,
+				"sensor reading changed for device %u: %08x\n",
+				i, value);
+
+		value = FIELD_GET(LPDDR2_MR4_SRR, value);
+		if (value > max)
+			max = value;
+	}
+
+	spin_unlock_irqrestore(&emc->lock, flags);
+
+	return max;
+}
+
+static void tegra210_emc_poll_refresh(struct timer_list *timer)
+{
+	struct tegra210_emc *emc = from_timer(emc, timer, refresh_timer);
+	unsigned int temperature;
+
+	if (!emc->debugfs.temperature)
+		temperature = tegra210_emc_get_temperature(emc);
+	else
+		temperature = emc->debugfs.temperature;
+
+	if (temperature == emc->temperature)
+		goto reset;
+
+	switch (temperature) {
+	case 0 ... 3:
+		/* temperature is fine, using regular refresh */
+		dev_dbg(emc->dev, "switching to nominal refresh...\n");
+		tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_NOMINAL);
+		break;
+
+	case 4:
+		dev_dbg(emc->dev, "switching to 2x refresh...\n");
+		tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_2X);
+		break;
+
+	case 5:
+		dev_dbg(emc->dev, "switching to 4x refresh...\n");
+		tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_4X);
+		break;
+
+	case 6 ... 7:
+		dev_dbg(emc->dev, "switching to throttle refresh...\n");
+		tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_THROTTLE);
+		break;
+
+	default:
+		WARN(1, "invalid DRAM temperature state %u\n", temperature);
+		return;
+	}
+
+	emc->temperature = temperature;
+
+reset:
+	if (atomic_read(&emc->refresh_poll) > 0) {
+		unsigned int interval = emc->refresh_poll_interval;
+		unsigned int timeout = msecs_to_jiffies(interval);
+
+		mod_timer(&emc->refresh_timer, jiffies + timeout);
+	}
+}
+
+static void tegra210_emc_poll_refresh_stop(struct tegra210_emc *emc)
+{
+	atomic_set(&emc->refresh_poll, 0);
+	del_timer_sync(&emc->refresh_timer);
+}
+
+static void tegra210_emc_poll_refresh_start(struct tegra210_emc *emc)
+{
+	atomic_set(&emc->refresh_poll, 1);
+
+	mod_timer(&emc->refresh_timer,
+		  jiffies + msecs_to_jiffies(emc->refresh_poll_interval));
+}
+
+static int tegra210_emc_cd_max_state(struct thermal_cooling_device *cd,
+				     unsigned long *state)
+{
+	*state = 1;
+
+	return 0;
+}
+
+static int tegra210_emc_cd_get_state(struct thermal_cooling_device *cd,
+				     unsigned long *state)
+{
+	struct tegra210_emc *emc = cd->devdata;
+
+	*state = atomic_read(&emc->refresh_poll);
+
+	return 0;
+}
+
+static int tegra210_emc_cd_set_state(struct thermal_cooling_device *cd,
+				     unsigned long state)
+{
+	struct tegra210_emc *emc = cd->devdata;
+
+	if (state == atomic_read(&emc->refresh_poll))
+		return 0;
+
+	if (state)
+		tegra210_emc_poll_refresh_start(emc);
+	else
+		tegra210_emc_poll_refresh_stop(emc);
+
+	return 0;
+}
+
+static struct thermal_cooling_device_ops tegra210_emc_cd_ops = {
+	.get_max_state = tegra210_emc_cd_max_state,
+	.get_cur_state = tegra210_emc_cd_get_state,
+	.set_cur_state = tegra210_emc_cd_set_state,
+};
+
 static void tegra210_emc_set_clock(struct tegra210_emc *emc, u32 clksrc)
 {
 	emc->sequence->set_clock(emc, clksrc);
@@ -628,6 +764,54 @@ static void tegra210_change_dll_src(struct tegra210_emc *emc,
 		tegra210_clk_emc_dll_enable(false);
 }
 
+int tegra210_emc_set_refresh(struct tegra210_emc *emc,
+			     enum tegra210_emc_refresh refresh)
+{
+	struct tegra210_emc_timing *timings;
+	unsigned long flags;
+
+	if ((emc->dram_type != DRAM_TYPE_LPDDR2 &&
+	     emc->dram_type != DRAM_TYPE_LPDDR4) ||
+	    !emc->last)
+		return -ENODEV;
+
+	if (refresh > TEGRA210_EMC_REFRESH_THROTTLE)
+		return -EINVAL;
+
+	if (refresh == emc->refresh)
+		return 0;
+
+	spin_lock_irqsave(&emc->lock, flags);
+
+	if (refresh == TEGRA210_EMC_REFRESH_THROTTLE && emc->derated)
+		timings = emc->derated;
+	else
+		timings = emc->nominal;
+
+	if (timings != emc->timings) {
+		unsigned int index = emc->last - emc->timings;
+		u32 clksrc;
+
+		clksrc = emc->provider.configs[index].value |
+			 EMC_CLK_FORCE_CC_TRIGGER;
+
+		emc->next = &timings[index];
+		emc->timings = timings;
+
+		tegra210_emc_set_clock(emc, clksrc);
+	} else {
+		tegra210_emc_adjust_timing(emc, emc->last);
+		tegra210_emc_timing_update(emc);
+
+		if (refresh != TEGRA210_EMC_REFRESH_NOMINAL)
+			emc_writel(emc, EMC_REF_REF_CMD, EMC_REF);
+	}
+
+	spin_unlock_irqrestore(&emc->lock, flags);
+
+	return 0;
+}
+
 u32 tegra210_emc_mrr_read(struct tegra210_emc *emc, unsigned int chip,
 			  unsigned int address)
 {
@@ -1306,6 +1490,42 @@ void tegra210_emc_dll_enable(struct tegra210_emc *emc)
 	update_dll_control(emc, value, true);
 }
 
+void tegra210_emc_adjust_timing(struct tegra210_emc *emc,
+				struct tegra210_emc_timing *timing)
+{
+	u32 dsr_cntrl = timing->burst_regs[EMC_DYN_SELF_REF_CONTROL_INDEX];
+	u32 pre_ref = timing->burst_regs[EMC_PRE_REFRESH_REQ_CNT_INDEX];
+	u32 ref = timing->burst_regs[EMC_REFRESH_INDEX];
+
+	switch (emc->refresh) {
+	case TEGRA210_EMC_REFRESH_NOMINAL:
+	case TEGRA210_EMC_REFRESH_THROTTLE:
+		break;
+
+	case TEGRA210_EMC_REFRESH_2X:
+		ref = REFRESH_SPEEDUP(ref, 2);
+		pre_ref = REFRESH_SPEEDUP(pre_ref, 2);
+		dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 2);
+		break;
+
+	case TEGRA210_EMC_REFRESH_4X:
+		ref = REFRESH_SPEEDUP(ref, 4);
+		pre_ref = REFRESH_SPEEDUP(pre_ref, 4);
+		dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 4);
+		break;
+
+	default:
+		dev_warn(emc->dev, "failed to set refresh: %d\n", emc->refresh);
+		return;
+	}
+
+	emc_writel(emc, ref, emc->offsets->burst[EMC_REFRESH_INDEX]);
+	emc_writel(emc, pre_ref,
+		   emc->offsets->burst[EMC_PRE_REFRESH_REQ_CNT_INDEX]);
+	emc_writel(emc, dsr_cntrl,
+		   emc->offsets->burst[EMC_DYN_SELF_REF_CONTROL_INDEX]);
+}
+
 static int tegra210_emc_set_rate(struct device *dev,
 				 const struct tegra210_clk_emc_config *config)
 {
@@ -1477,6 +1697,37 @@ DEFINE_SIMPLE_ATTRIBUTE(tegra210_emc_debug_max_rate_fops,
 			tegra210_emc_debug_max_rate_get,
 			tegra210_emc_debug_max_rate_set, "%llu\n");
 
+static int tegra210_emc_debug_temperature_get(void *data, u64 *temperature)
+{
+	struct tegra210_emc *emc = data;
+	unsigned int value;
+
+	if (!emc->debugfs.temperature)
+		value = tegra210_emc_get_temperature(emc);
+	else
+		value = emc->debugfs.temperature;
+
+	*temperature = value;
+
+	return 0;
+}
+
+static int tegra210_emc_debug_temperature_set(void *data, u64 temperature)
+{
+	struct tegra210_emc *emc = data;
+
+	if (temperature > 7)
+		return -EINVAL;
+
+	emc->debugfs.temperature = temperature;
+
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(tegra210_emc_debug_temperature_fops,
+			tegra210_emc_debug_temperature_get,
+			tegra210_emc_debug_temperature_set, "%llu\n");
+
 static void tegra210_emc_debugfs_init(struct tegra210_emc *emc)
 {
 	struct device *dev = emc->dev;
@@ -1520,6 +1771,8 @@ static void tegra210_emc_debugfs_init(struct tegra210_emc *emc)
 			    &tegra210_emc_debug_min_rate_fops);
 	debugfs_create_file("max_rate", 0644, emc->debugfs.root, emc,
 			    &tegra210_emc_debug_max_rate_fops);
+	debugfs_create_file("temperature", 0644, emc->debugfs.root, emc,
+			    &tegra210_emc_debug_temperature_fops);
 }
 
 static void tegra210_emc_detect(struct tegra210_emc *emc)
@@ -1574,6 +1827,7 @@ static int tegra210_emc_validate_timings(struct tegra210_emc *emc,
 
 static int tegra210_emc_probe(struct platform_device *pdev)
 {
+	struct thermal_cooling_device *cd;
 	unsigned long current_rate;
 	struct platform_device *mc;
 	struct tegra210_emc *emc;
@@ -1627,17 +1881,36 @@ static int tegra210_emc_probe(struct platform_device *pdev)
 	tegra210_emc_detect(emc);
 	np = pdev->dev.of_node;
 
-	err = of_reserved_mem_device_init(emc->dev);
+	/* attach to the nominal and (optional) derated tables */
+	err = of_reserved_mem_device_init_by_name(emc->dev, np, "nominal");
 	if (err < 0) {
-		dev_err(&pdev->dev, "failed to get EMC table: %d\n", err);
+		dev_err(emc->dev, "failed to get nominal EMC table: %d\n", err);
 		goto put_mc;
 	}
 
+	err = of_reserved_mem_device_init_by_name(emc->dev, np, "derated");
+	if (err < 0 && err != -ENODEV) {
+		dev_err(emc->dev, "failed to get derated EMC table: %d\n", err);
+		goto release;
+	}
+
 	/* validate the tables */
-	err = tegra210_emc_validate_timings(emc, emc->timings,
+	if (emc->nominal) {
+		err = tegra210_emc_validate_timings(emc, emc->nominal,
 						    emc->num_timings);
 		if (err < 0)
 			goto release;
+	}
+
+	if (emc->derated) {
+		err = tegra210_emc_validate_timings(emc, emc->derated,
+						    emc->num_timings);
+		if (err < 0)
+			goto release;
+	}
+
+	/* default to the nominal table */
+	emc->timings = emc->nominal;
 
 	/* pick the current timing based on the current EMC clock rate */
 	current_rate = clk_get_rate(emc->clk) / 1000;
@@ -1675,6 +1948,7 @@ static int tegra210_emc_probe(struct platform_device *pdev)
 	}
 
 	emc->offsets = &tegra210_emc_table_register_offsets;
+	emc->refresh = TEGRA210_EMC_REFRESH_NOMINAL;
 
 	emc->provider.owner = THIS_MODULE;
 	emc->provider.dev = &pdev->dev;
@@ -1717,12 +1991,29 @@ static int tegra210_emc_probe(struct platform_device *pdev)
 	emc->training_interval = 100;
 	dev_set_drvdata(emc->dev, emc);
 
+	timer_setup(&emc->refresh_timer, tegra210_emc_poll_refresh,
+		    TIMER_DEFERRABLE);
+	atomic_set(&emc->refresh_poll, 0);
+	emc->refresh_poll_interval = 1000;
+
 	timer_setup(&emc->training, tegra210_emc_train, 0);
 
 	tegra210_emc_debugfs_init(emc);
 
+	cd = devm_thermal_of_cooling_device_register(emc->dev, np, "emc", emc,
+						     &tegra210_emc_cd_ops);
+	if (IS_ERR(cd)) {
+		err = PTR_ERR(cd);
+		dev_err(emc->dev, "failed to register cooling device: %d\n",
+			err);
+		goto detach;
+	}
+
 	return 0;
 
+detach:
+	debugfs_remove_recursive(emc->debugfs.root);
+	tegra210_clk_emc_detach(emc->clk);
 release:
 	of_reserved_mem_device_release(emc->dev);
 put_mc:

drivers/memory/tegra/tegra210-emc-table.c

@@ -13,32 +13,63 @@ static int tegra210_emc_table_device_init(struct reserved_mem *rmem,
 					  struct device *dev)
 {
 	struct tegra210_emc *emc = dev_get_drvdata(dev);
-	unsigned int i;
+	struct tegra210_emc_timing *timings;
+	unsigned int i, count = 0;
 
-	emc->timings = memremap(rmem->base, rmem->size, MEMREMAP_WB);
-	if (!emc->timings) {
+	timings = memremap(rmem->base, rmem->size, MEMREMAP_WB);
+	if (!timings) {
 		dev_err(dev, "failed to map EMC table\n");
 		return -ENOMEM;
 	}
 
-	emc->num_timings = 0;
+	count = 0;
 
 	for (i = 0; i < TEGRA_EMC_MAX_FREQS; i++) {
-		if (emc->timings[i].revision == 0)
+		if (timings[i].revision == 0)
 			break;
 
-		emc->num_timings++;
+		count++;
 	}
 
+	/* only the nominal and derated tables are expected */
+	if (emc->derated) {
+		dev_warn(dev, "excess EMC table '%s'\n", rmem->name);
+		goto out;
+	}
+
+	if (emc->nominal) {
+		if (count != emc->num_timings) {
+			dev_warn(dev, "%u derated vs. %u nominal entries\n",
+				 count, emc->num_timings);
+			memunmap(timings);
+			return -EINVAL;
+		}
+
+		emc->derated = timings;
+	} else {
+		emc->num_timings = count;
+		emc->nominal = timings;
+	}
+
+out:
+	/* keep track of which table this is */
+	rmem->priv = timings;
+
 	return 0;
 }
 
 static void tegra210_emc_table_device_release(struct reserved_mem *rmem,
 					      struct device *dev)
 {
+	struct tegra210_emc_timing *timings = rmem->priv;
 	struct tegra210_emc *emc = dev_get_drvdata(dev);
 
-	memunmap(emc->timings);
+	if ((emc->nominal && timings != emc->nominal) &&
+	    (emc->derated && timings != emc->derated))
+		dev_warn(dev, "trying to release unassigned EMC table '%s'\n",
+			 rmem->name);
+
+	memunmap(timings);
 }
 
 static const struct reserved_mem_ops tegra210_emc_table_ops = {

drivers/memory/tegra/tegra210-emc.h

@@ -83,6 +83,7 @@
 #define EMC_EMRS						0xd0
 #define EMC_EMRS_USE_EMRS_LONG_CNT				BIT(26)
 #define EMC_REF							0xd4
+#define  EMC_REF_REF_CMD					BIT(0)
 #define EMC_SELF_REF						0xe0
 #define EMC_MRW							0xe8
 #define EMC_MRW_MRW_OP_SHIFT					0
@@ -871,6 +872,13 @@ struct tegra210_emc_timing {
 	u32 latency;
 };
 
+enum tegra210_emc_refresh {
+	TEGRA210_EMC_REFRESH_NOMINAL = 0,
+	TEGRA210_EMC_REFRESH_2X,
+	TEGRA210_EMC_REFRESH_4X,
+	TEGRA210_EMC_REFRESH_THROTTLE, /* 4x Refresh + derating. */
+};
+
 #define DRAM_TYPE_DDR3		0
 #define DRAM_TYPE_LPDDR4	1
 #define DRAM_TYPE_LPDDR2	2
@@ -881,6 +889,12 @@ struct tegra210_emc {
 	struct device *dev;
 	struct clk *clk;
 
+	/* nominal EMC frequency table */
+	struct tegra210_emc_timing *nominal;
+	/* derated EMC frequency table */
+	struct tegra210_emc_timing *derated;
+
+	/* currently selected table (nominal or derated) */
 	struct tegra210_emc_timing *timings;
 	unsigned int num_timings;
 
@@ -900,6 +914,12 @@ struct tegra210_emc {
 	unsigned int training_interval;
 	struct timer_list training;
 
+	enum tegra210_emc_refresh refresh;
+	unsigned int refresh_poll_interval;
+	struct timer_list refresh_timer;
+	unsigned int temperature;
+	atomic_t refresh_poll;
+
 	ktime_t clkchange_time;
 	int clkchange_delay;
 
@@ -909,6 +929,7 @@ struct tegra210_emc {
 		struct dentry *root;
 		unsigned long min_rate;
 		unsigned long max_rate;
+		unsigned int temperature;
 	} debugfs;
 
 	struct tegra210_clk_emc_provider provider;
@@ -967,6 +988,8 @@ static inline u32 div_o3(u32 a, u32 b)
 /* from tegra210-emc-r21021.c */
 extern const struct tegra210_emc_sequence tegra210_emc_r21021;
 
+int tegra210_emc_set_refresh(struct tegra210_emc *emc,
+			     enum tegra210_emc_refresh refresh);
 u32 tegra210_emc_mrr_read(struct tegra210_emc *emc, unsigned int chip,
 			  unsigned int address);
 void tegra210_emc_do_clock_change(struct tegra210_emc *emc, u32 clksrc);
@@ -975,6 +998,8 @@ void tegra210_emc_timing_update(struct tegra210_emc *emc);
 u32 tegra210_emc_get_dll_state(struct tegra210_emc_timing *next);
 struct tegra210_emc_timing *tegra210_emc_find_timing(struct tegra210_emc *emc,
 						     unsigned long rate);
+void tegra210_emc_adjust_timing(struct tegra210_emc *emc,
+				struct tegra210_emc_timing *timing);
 int tegra210_emc_wait_for_update(struct tegra210_emc *emc, unsigned int channel,
 				 unsigned int offset, u32 bit_mask, bool state);
 unsigned long tegra210_emc_actual_osc_clocks(u32 in);