Commit c6570114 authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'rproc-v5.7' of git://git.kernel.org/pub/scm/linux/kernel/git/andersson/remoteproc

Pull remoteproc updates from Bjorn Andersson:

 - a range of improvements to the OMAP remoeteproc driver; among other
   things adding devicetree, suspend/resume and watchdog support, and
   adds support the remoteprocs in the DRA7xx SoC

 - support for 64-bit firmware, extends the ELF loader to support this
   and fixes for a number of race conditions in the recovery handling

 - a generic mechanism to allow remoteproc drivers to sync state with
   remote processors during a panic, and uses this to prepare Qualcomm
   remote processors for post mortem analysis

 - fixes to cleanly recover from crashes in the modem firmware on
   production Qualcomm devices

* tag 'rproc-v5.7' of git://git.kernel.org/pub/scm/linux/kernel/git/andersson/remoteproc: (37 commits)
  remoteproc/omap: Switch to SPDX license identifiers
  remoteproc/omap: Add watchdog functionality for remote processors
  remoteproc/omap: Report device exceptions and trigger recovery
  remoteproc/omap: Add support for runtime auto-suspend/resume
  remoteproc/omap: Add support for system suspend/resume
  remoteproc/omap: Request a timer(s) for remoteproc usage
  remoteproc/omap: Check for undefined mailbox messages
  remoteproc/omap: Remove the platform_data header
  remoteproc/omap: Add support for DRA7xx remote processors
  remoteproc/omap: Initialize and assign reserved memory node
  remoteproc/omap: Add the rproc ops .da_to_va() implementation
  remoteproc/omap: Add support to parse internal memories from DT
  remoteproc/omap: Add a sanity check for DSP boot address alignment
  remoteproc/omap: Add device tree support
  dt-bindings: remoteproc: Add OMAP remoteproc bindings
  remoteproc: qcom: Introduce panic handler for PAS and ADSP
  remoteproc: qcom: q6v5: Add common panic handler
  remoteproc: Introduce "panic" callback in ops
  remoteproc: Traverse rproc_list under RCU read lock
  remoteproc: Fix NULL pointer dereference in rproc_virtio_notify
  ...
parents ac438771 a7084c3d
...@@ -230,7 +230,7 @@ in the used rings. ...@@ -230,7 +230,7 @@ in the used rings.
Binary Firmware Structure Binary Firmware Structure
========================= =========================
At this point remoteproc only supports ELF32 firmware binaries. However, At this point remoteproc supports ELF32 and ELF64 firmware binaries. However,
it is quite expected that other platforms/devices which we'd want to it is quite expected that other platforms/devices which we'd want to
support with this framework will be based on different binary formats. support with this framework will be based on different binary formats.
......
...@@ -35,7 +35,7 @@ config MTK_SCP ...@@ -35,7 +35,7 @@ config MTK_SCP
config OMAP_REMOTEPROC config OMAP_REMOTEPROC
tristate "OMAP remoteproc support" tristate "OMAP remoteproc support"
depends on ARCH_OMAP4 || SOC_OMAP5 depends on ARCH_OMAP4 || SOC_OMAP5 || SOC_DRA7XX
depends on OMAP_IOMMU depends on OMAP_IOMMU
select MAILBOX select MAILBOX
select OMAP2PLUS_MBOX select OMAP2PLUS_MBOX
...@@ -52,6 +52,18 @@ config OMAP_REMOTEPROC ...@@ -52,6 +52,18 @@ config OMAP_REMOTEPROC
It's safe to say N here if you're not interested in multimedia It's safe to say N here if you're not interested in multimedia
offloading or just want a bare minimum kernel. offloading or just want a bare minimum kernel.
config OMAP_REMOTEPROC_WATCHDOG
bool "OMAP remoteproc watchdog timer"
depends on OMAP_REMOTEPROC
default n
help
Say Y here to enable watchdog timer for remote processors.
This option controls the watchdog functionality for the remote
processors in OMAP. Dedicated OMAP DMTimers are used by the remote
processors and triggers the timer interrupt upon a watchdog
detection.
config WKUP_M3_RPROC config WKUP_M3_RPROC
tristate "AMx3xx Wakeup M3 remoteproc support" tristate "AMx3xx Wakeup M3 remoteproc support"
depends on SOC_AM33XX || SOC_AM43XX depends on SOC_AM33XX || SOC_AM43XX
......
...@@ -186,7 +186,7 @@ static int imx_rproc_stop(struct rproc *rproc) ...@@ -186,7 +186,7 @@ static int imx_rproc_stop(struct rproc *rproc)
} }
static int imx_rproc_da_to_sys(struct imx_rproc *priv, u64 da, static int imx_rproc_da_to_sys(struct imx_rproc *priv, u64 da,
int len, u64 *sys) size_t len, u64 *sys)
{ {
const struct imx_rproc_dcfg *dcfg = priv->dcfg; const struct imx_rproc_dcfg *dcfg = priv->dcfg;
int i; int i;
...@@ -203,19 +203,19 @@ static int imx_rproc_da_to_sys(struct imx_rproc *priv, u64 da, ...@@ -203,19 +203,19 @@ static int imx_rproc_da_to_sys(struct imx_rproc *priv, u64 da,
} }
} }
dev_warn(priv->dev, "Translation failed: da = 0x%llx len = 0x%x\n", dev_warn(priv->dev, "Translation failed: da = 0x%llx len = 0x%zx\n",
da, len); da, len);
return -ENOENT; return -ENOENT;
} }
static void *imx_rproc_da_to_va(struct rproc *rproc, u64 da, int len) static void *imx_rproc_da_to_va(struct rproc *rproc, u64 da, size_t len)
{ {
struct imx_rproc *priv = rproc->priv; struct imx_rproc *priv = rproc->priv;
void *va = NULL; void *va = NULL;
u64 sys; u64 sys;
int i; int i;
if (len <= 0) if (len == 0)
return NULL; return NULL;
/* /*
...@@ -235,7 +235,8 @@ static void *imx_rproc_da_to_va(struct rproc *rproc, u64 da, int len) ...@@ -235,7 +235,8 @@ static void *imx_rproc_da_to_va(struct rproc *rproc, u64 da, int len)
} }
} }
dev_dbg(&rproc->dev, "da = 0x%llx len = 0x%x va = 0x%p\n", da, len, va); dev_dbg(&rproc->dev, "da = 0x%llx len = 0x%zx va = 0x%p\n",
da, len, va);
return va; return va;
} }
......
...@@ -246,7 +246,7 @@ static void keystone_rproc_kick(struct rproc *rproc, int vqid) ...@@ -246,7 +246,7 @@ static void keystone_rproc_kick(struct rproc *rproc, int vqid)
* can be used either by the remoteproc core for loading (when using kernel * can be used either by the remoteproc core for loading (when using kernel
* remoteproc loader), or by any rpmsg bus drivers. * remoteproc loader), or by any rpmsg bus drivers.
*/ */
static void *keystone_rproc_da_to_va(struct rproc *rproc, u64 da, int len) static void *keystone_rproc_da_to_va(struct rproc *rproc, u64 da, size_t len)
{ {
struct keystone_rproc *ksproc = rproc->priv; struct keystone_rproc *ksproc = rproc->priv;
void __iomem *va = NULL; void __iomem *va = NULL;
...@@ -255,7 +255,7 @@ static void *keystone_rproc_da_to_va(struct rproc *rproc, u64 da, int len) ...@@ -255,7 +255,7 @@ static void *keystone_rproc_da_to_va(struct rproc *rproc, u64 da, int len)
size_t size; size_t size;
int i; int i;
if (len <= 0) if (len == 0)
return NULL; return NULL;
for (i = 0; i < ksproc->num_mems; i++) { for (i = 0; i < ksproc->num_mems; i++) {
......
...@@ -320,7 +320,7 @@ static int scp_start(struct rproc *rproc) ...@@ -320,7 +320,7 @@ static int scp_start(struct rproc *rproc)
return ret; return ret;
} }
static void *scp_da_to_va(struct rproc *rproc, u64 da, int len) static void *scp_da_to_va(struct rproc *rproc, u64 da, size_t len)
{ {
struct mtk_scp *scp = (struct mtk_scp *)rproc->priv; struct mtk_scp *scp = (struct mtk_scp *)rproc->priv;
int offset; int offset;
......
This diff is collapsed.
/* SPDX-License-Identifier: BSD-3-Clause */
/* /*
* Remote processor messaging * Remote processor messaging
* *
* Copyright (C) 2011 Texas Instruments, Inc. * Copyright (C) 2011-2020 Texas Instruments, Inc.
* Copyright (C) 2011 Google, Inc. * Copyright (C) 2011 Google, Inc.
* All rights reserved. * All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name Texas Instruments nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
#ifndef _OMAP_RPMSG_H #ifndef _OMAP_RPMSG_H
...@@ -56,6 +31,22 @@ ...@@ -56,6 +31,22 @@
* *
* @RP_MBOX_ABORT_REQUEST: a "please crash" request, used for testing the * @RP_MBOX_ABORT_REQUEST: a "please crash" request, used for testing the
* recovery mechanism (to some extent). * recovery mechanism (to some extent).
*
* @RP_MBOX_SUSPEND_AUTO: auto suspend request for the remote processor
*
* @RP_MBOX_SUSPEND_SYSTEM: system suspend request for the remote processor
*
* @RP_MBOX_SUSPEND_ACK: successful response from remote processor for a
* suspend request
*
* @RP_MBOX_SUSPEND_CANCEL: a cancel suspend response from a remote processor
* on a suspend request
*
* Introduce new message definitions if any here.
*
* @RP_MBOX_END_MSG: Indicates end of known/defined messages from remote core
* This should be the last definition.
*
*/ */
enum omap_rp_mbox_messages { enum omap_rp_mbox_messages {
RP_MBOX_READY = 0xFFFFFF00, RP_MBOX_READY = 0xFFFFFF00,
...@@ -64,6 +55,11 @@ enum omap_rp_mbox_messages { ...@@ -64,6 +55,11 @@ enum omap_rp_mbox_messages {
RP_MBOX_ECHO_REQUEST = 0xFFFFFF03, RP_MBOX_ECHO_REQUEST = 0xFFFFFF03,
RP_MBOX_ECHO_REPLY = 0xFFFFFF04, RP_MBOX_ECHO_REPLY = 0xFFFFFF04,
RP_MBOX_ABORT_REQUEST = 0xFFFFFF05, RP_MBOX_ABORT_REQUEST = 0xFFFFFF05,
RP_MBOX_SUSPEND_AUTO = 0xFFFFFF10,
RP_MBOX_SUSPEND_SYSTEM = 0xFFFFFF11,
RP_MBOX_SUSPEND_ACK = 0xFFFFFF12,
RP_MBOX_SUSPEND_CANCEL = 0xFFFFFF13,
RP_MBOX_END_MSG = 0xFFFFFF14,
}; };
#endif /* _OMAP_RPMSG_H */ #endif /* _OMAP_RPMSG_H */
...@@ -15,6 +15,8 @@ ...@@ -15,6 +15,8 @@
#include <linux/remoteproc.h> #include <linux/remoteproc.h>
#include "qcom_q6v5.h" #include "qcom_q6v5.h"
#define Q6V5_PANIC_DELAY_MS 200
/** /**
* qcom_q6v5_prepare() - reinitialize the qcom_q6v5 context before start * qcom_q6v5_prepare() - reinitialize the qcom_q6v5 context before start
* @q6v5: reference to qcom_q6v5 context to be reinitialized * @q6v5: reference to qcom_q6v5 context to be reinitialized
...@@ -162,6 +164,24 @@ int qcom_q6v5_request_stop(struct qcom_q6v5 *q6v5) ...@@ -162,6 +164,24 @@ int qcom_q6v5_request_stop(struct qcom_q6v5 *q6v5)
} }
EXPORT_SYMBOL_GPL(qcom_q6v5_request_stop); EXPORT_SYMBOL_GPL(qcom_q6v5_request_stop);
/**
* qcom_q6v5_panic() - panic handler to invoke a stop on the remote
* @q6v5: reference to qcom_q6v5 context
*
* Set the stop bit and sleep in order to allow the remote processor to flush
* its caches etc for post mortem debugging.
*
* Return: 200ms
*/
unsigned long qcom_q6v5_panic(struct qcom_q6v5 *q6v5)
{
qcom_smem_state_update_bits(q6v5->state,
BIT(q6v5->stop_bit), BIT(q6v5->stop_bit));
return Q6V5_PANIC_DELAY_MS;
}
EXPORT_SYMBOL_GPL(qcom_q6v5_panic);
/** /**
* qcom_q6v5_init() - initializer of the q6v5 common struct * qcom_q6v5_init() - initializer of the q6v5 common struct
* @q6v5: handle to be initialized * @q6v5: handle to be initialized
......
...@@ -42,5 +42,6 @@ int qcom_q6v5_prepare(struct qcom_q6v5 *q6v5); ...@@ -42,5 +42,6 @@ int qcom_q6v5_prepare(struct qcom_q6v5 *q6v5);
int qcom_q6v5_unprepare(struct qcom_q6v5 *q6v5); int qcom_q6v5_unprepare(struct qcom_q6v5 *q6v5);
int qcom_q6v5_request_stop(struct qcom_q6v5 *q6v5); int qcom_q6v5_request_stop(struct qcom_q6v5 *q6v5);
int qcom_q6v5_wait_for_start(struct qcom_q6v5 *q6v5, int timeout); int qcom_q6v5_wait_for_start(struct qcom_q6v5 *q6v5, int timeout);
unsigned long qcom_q6v5_panic(struct qcom_q6v5 *q6v5);
#endif #endif
...@@ -270,7 +270,7 @@ static int adsp_stop(struct rproc *rproc) ...@@ -270,7 +270,7 @@ static int adsp_stop(struct rproc *rproc)
return ret; return ret;
} }
static void *adsp_da_to_va(struct rproc *rproc, u64 da, int len) static void *adsp_da_to_va(struct rproc *rproc, u64 da, size_t len)
{ {
struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv; struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
int offset; int offset;
...@@ -282,12 +282,20 @@ static void *adsp_da_to_va(struct rproc *rproc, u64 da, int len) ...@@ -282,12 +282,20 @@ static void *adsp_da_to_va(struct rproc *rproc, u64 da, int len)
return adsp->mem_region + offset; return adsp->mem_region + offset;
} }
static unsigned long adsp_panic(struct rproc *rproc)
{
struct qcom_adsp *adsp = rproc->priv;
return qcom_q6v5_panic(&adsp->q6v5);
}
static const struct rproc_ops adsp_ops = { static const struct rproc_ops adsp_ops = {
.start = adsp_start, .start = adsp_start,
.stop = adsp_stop, .stop = adsp_stop,
.da_to_va = adsp_da_to_va, .da_to_va = adsp_da_to_va,
.parse_fw = qcom_register_dump_segments, .parse_fw = qcom_register_dump_segments,
.load = adsp_load, .load = adsp_load,
.panic = adsp_panic,
}; };
static int adsp_init_clock(struct qcom_adsp *adsp, const char **clk_ids) static int adsp_init_clock(struct qcom_adsp *adsp, const char **clk_ids)
......
...@@ -381,23 +381,33 @@ static void q6v5_pds_disable(struct q6v5 *qproc, struct device **pds, ...@@ -381,23 +381,33 @@ static void q6v5_pds_disable(struct q6v5 *qproc, struct device **pds,
} }
static int q6v5_xfer_mem_ownership(struct q6v5 *qproc, int *current_perm, static int q6v5_xfer_mem_ownership(struct q6v5 *qproc, int *current_perm,
bool remote_owner, phys_addr_t addr, bool local, bool remote, phys_addr_t addr,
size_t size) size_t size)
{ {
struct qcom_scm_vmperm next; struct qcom_scm_vmperm next[2];
int perms = 0;
if (!qproc->need_mem_protection) if (!qproc->need_mem_protection)
return 0; return 0;
if (remote_owner && *current_perm == BIT(QCOM_SCM_VMID_MSS_MSA))
return 0; if (local == !!(*current_perm & BIT(QCOM_SCM_VMID_HLOS)) &&
if (!remote_owner && *current_perm == BIT(QCOM_SCM_VMID_HLOS)) remote == !!(*current_perm & BIT(QCOM_SCM_VMID_MSS_MSA)))
return 0; return 0;
next.vmid = remote_owner ? QCOM_SCM_VMID_MSS_MSA : QCOM_SCM_VMID_HLOS; if (local) {
next.perm = remote_owner ? QCOM_SCM_PERM_RW : QCOM_SCM_PERM_RWX; next[perms].vmid = QCOM_SCM_VMID_HLOS;
next[perms].perm = QCOM_SCM_PERM_RWX;
perms++;
}
if (remote) {
next[perms].vmid = QCOM_SCM_VMID_MSS_MSA;
next[perms].perm = QCOM_SCM_PERM_RW;
perms++;
}
return qcom_scm_assign_mem(addr, ALIGN(size, SZ_4K), return qcom_scm_assign_mem(addr, ALIGN(size, SZ_4K),
current_perm, &next, 1); current_perm, next, perms);
} }
static int q6v5_load(struct rproc *rproc, const struct firmware *fw) static int q6v5_load(struct rproc *rproc, const struct firmware *fw)
...@@ -803,7 +813,8 @@ static int q6v5_mpss_init_image(struct q6v5 *qproc, const struct firmware *fw) ...@@ -803,7 +813,8 @@ static int q6v5_mpss_init_image(struct q6v5 *qproc, const struct firmware *fw)
/* Hypervisor mapping to access metadata by modem */ /* Hypervisor mapping to access metadata by modem */
mdata_perm = BIT(QCOM_SCM_VMID_HLOS); mdata_perm = BIT(QCOM_SCM_VMID_HLOS);
ret = q6v5_xfer_mem_ownership(qproc, &mdata_perm, true, phys, size); ret = q6v5_xfer_mem_ownership(qproc, &mdata_perm, false, true,
phys, size);
if (ret) { if (ret) {
dev_err(qproc->dev, dev_err(qproc->dev,
"assigning Q6 access to metadata failed: %d\n", ret); "assigning Q6 access to metadata failed: %d\n", ret);
...@@ -821,7 +832,8 @@ static int q6v5_mpss_init_image(struct q6v5 *qproc, const struct firmware *fw) ...@@ -821,7 +832,8 @@ static int q6v5_mpss_init_image(struct q6v5 *qproc, const struct firmware *fw)
dev_err(qproc->dev, "MPSS header authentication failed: %d\n", ret); dev_err(qproc->dev, "MPSS header authentication failed: %d\n", ret);
/* Metadata authentication done, remove modem access */ /* Metadata authentication done, remove modem access */
xferop_ret = q6v5_xfer_mem_ownership(qproc, &mdata_perm, false, phys, size); xferop_ret = q6v5_xfer_mem_ownership(qproc, &mdata_perm, true, false,
phys, size);
if (xferop_ret) if (xferop_ret)
dev_warn(qproc->dev, dev_warn(qproc->dev,
"mdt buffer not reclaimed system may become unstable\n"); "mdt buffer not reclaimed system may become unstable\n");
...@@ -908,7 +920,7 @@ static int q6v5_mba_load(struct q6v5 *qproc) ...@@ -908,7 +920,7 @@ static int q6v5_mba_load(struct q6v5 *qproc)
} }
/* Assign MBA image access in DDR to q6 */ /* Assign MBA image access in DDR to q6 */
ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, true, ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, false, true,
qproc->mba_phys, qproc->mba_size); qproc->mba_phys, qproc->mba_size);
if (ret) { if (ret) {
dev_err(qproc->dev, dev_err(qproc->dev,
...@@ -945,8 +957,8 @@ static int q6v5_mba_load(struct q6v5 *qproc) ...@@ -945,8 +957,8 @@ static int q6v5_mba_load(struct q6v5 *qproc)
q6v5proc_halt_axi_port(qproc, qproc->halt_map, qproc->halt_nc); q6v5proc_halt_axi_port(qproc, qproc->halt_map, qproc->halt_nc);
reclaim_mba: reclaim_mba:
xfermemop_ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, false, xfermemop_ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, true,
qproc->mba_phys, false, qproc->mba_phys,
qproc->mba_size); qproc->mba_size);
if (xfermemop_ret) { if (xfermemop_ret) {
dev_err(qproc->dev, dev_err(qproc->dev,
...@@ -1003,11 +1015,6 @@ static void q6v5_mba_reclaim(struct q6v5 *qproc) ...@@ -1003,11 +1015,6 @@ static void q6v5_mba_reclaim(struct q6v5 *qproc)
writel(val, qproc->reg_base + QDSP6SS_PWR_CTL_REG); writel(val, qproc->reg_base + QDSP6SS_PWR_CTL_REG);
} }
ret = q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm,
false, qproc->mpss_phys,
qproc->mpss_size);
WARN_ON(ret);
q6v5_reset_assert(qproc); q6v5_reset_assert(qproc);
q6v5_clk_disable(qproc->dev, qproc->reset_clks, q6v5_clk_disable(qproc->dev, qproc->reset_clks,
...@@ -1021,7 +1028,7 @@ static void q6v5_mba_reclaim(struct q6v5 *qproc) ...@@ -1021,7 +1028,7 @@ static void q6v5_mba_reclaim(struct q6v5 *qproc)
/* In case of failure or coredump scenario where reclaiming MBA memory /* In case of failure or coredump scenario where reclaiming MBA memory
* could not happen reclaim it here. * could not happen reclaim it here.
*/ */
ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, false, ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, true, false,
qproc->mba_phys, qproc->mba_phys,
qproc->mba_size); qproc->mba_size);
WARN_ON(ret); WARN_ON(ret);
...@@ -1037,6 +1044,23 @@ static void q6v5_mba_reclaim(struct q6v5 *qproc) ...@@ -1037,6 +1044,23 @@ static void q6v5_mba_reclaim(struct q6v5 *qproc)
} }
} }
static int q6v5_reload_mba(struct rproc *rproc)
{
struct q6v5 *qproc = rproc->priv;
const struct firmware *fw;
int ret;
ret = request_firmware(&fw, rproc->firmware, qproc->dev);
if (ret < 0)
return ret;
q6v5_load(rproc, fw);
ret = q6v5_mba_load(qproc);
release_firmware(fw);
return ret;
}
static int q6v5_mpss_load(struct q6v5 *qproc) static int q6v5_mpss_load(struct q6v5 *qproc)
{ {
const struct elf32_phdr *phdrs; const struct elf32_phdr *phdrs;
...@@ -1048,6 +1072,7 @@ static int q6v5_mpss_load(struct q6v5 *qproc) ...@@ -1048,6 +1072,7 @@ static int q6v5_mpss_load(struct q6v5 *qproc)
phys_addr_t boot_addr; phys_addr_t boot_addr;
phys_addr_t min_addr = PHYS_ADDR_MAX; phys_addr_t min_addr = PHYS_ADDR_MAX;
phys_addr_t max_addr = 0; phys_addr_t max_addr = 0;
u32 code_length;
bool relocate = false; bool relocate = false;
char *fw_name; char *fw_name;
size_t fw_name_len; size_t fw_name_len;
...@@ -1097,6 +1122,24 @@ static int q6v5_mpss_load(struct q6v5 *qproc) ...@@ -1097,6 +1122,24 @@ static int q6v5_mpss_load(struct q6v5 *qproc)
max_addr = ALIGN(phdr->p_paddr + phdr->p_memsz, SZ_4K); max_addr = ALIGN(phdr->p_paddr + phdr->p_memsz, SZ_4K);
} }
/**
* In case of a modem subsystem restart on secure devices, the modem
* memory can be reclaimed only after MBA is loaded. For modem cold
* boot this will be a nop
*/
q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm, true, false,
qproc->mpss_phys, qproc->mpss_size);
/* Share ownership between Linux and MSS, during segment loading */
ret = q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm, true, true,
qproc->mpss_phys, qproc->mpss_size);
if (ret) {
dev_err(qproc->dev,
"assigning Q6 access to mpss memory failed: %d\n", ret);
ret = -EAGAIN;
goto release_firmware;
}
mpss_reloc = relocate ? min_addr : qproc->mpss_phys; mpss_reloc = relocate ? min_addr : qproc->mpss_phys;
qproc->mpss_reloc = mpss_reloc; qproc->mpss_reloc = mpss_reloc;
/* Load firmware segments */ /* Load firmware segments */
...@@ -1145,10 +1188,25 @@ static int q6v5_mpss_load(struct q6v5 *qproc) ...@@ -1145,10 +1188,25 @@ static int q6v5_mpss_load(struct q6v5 *qproc)
phdr->p_memsz - phdr->p_filesz); phdr->p_memsz - phdr->p_filesz);
} }
size += phdr->p_memsz; size += phdr->p_memsz;
code_length = readl(qproc->rmb_base + RMB_PMI_CODE_LENGTH_REG);
if (!code_length) {
boot_addr = relocate ? qproc->mpss_phys : min_addr;
writel(boot_addr, qproc->rmb_base + RMB_PMI_CODE_START_REG);
writel(RMB_CMD_LOAD_READY, qproc->rmb_base + RMB_MBA_COMMAND_REG);
}
writel(size, qproc->rmb_base + RMB_PMI_CODE_LENGTH_REG);
ret = readl(qproc->rmb_base + RMB_MBA_STATUS_REG);
if (ret < 0) {
dev_err(qproc->dev, "MPSS authentication failed: %d\n",
ret);
goto release_firmware;
}
} }
/* Transfer ownership of modem ddr region to q6 */ /* Transfer ownership of modem ddr region to q6 */
ret = q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm, true, ret = q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm, false, true,
qproc->mpss_phys, qproc->mpss_size); qproc->mpss_phys, qproc->mpss_size);
if (ret) { if (ret) {
dev_err(qproc->dev, dev_err(qproc->dev,
...@@ -1157,11 +1215,6 @@ static int q6v5_mpss_load(struct q6v5 *qproc) ...@@ -1157,11 +1215,6 @@ static int q6v5_mpss_load(struct q6v5 *qproc)
goto release_firmware; goto release_firmware;
} }
boot_addr = relocate ? qproc->mpss_phys : min_addr;
writel(boot_addr, qproc->rmb_base + RMB_PMI_CODE_START_REG);
writel(RMB_CMD_LOAD_READY, qproc->rmb_base + RMB_MBA_COMMAND_REG);
writel(size, qproc->rmb_base + RMB_PMI_CODE_LENGTH_REG);
ret = q6v5_rmb_mba_wait(qproc, RMB_MBA_AUTH_COMPLETE, 10000); ret = q6v5_rmb_mba_wait(qproc, RMB_MBA_AUTH_COMPLETE, 10000);
if (ret == -ETIMEDOUT) if (ret == -ETIMEDOUT)
dev_err(qproc->dev, "MPSS authentication timed out\n"); dev_err(qproc->dev, "MPSS authentication timed out\n");
...@@ -1186,8 +1239,16 @@ static void qcom_q6v5_dump_segment(struct rproc *rproc, ...@@ -1186,8 +1239,16 @@ static void qcom_q6v5_dump_segment(struct rproc *rproc,
void *ptr = rproc_da_to_va(rproc, segment->da, segment->size); void *ptr = rproc_da_to_va(rproc, segment->da, segment->size);
/* Unlock mba before copying segments */ /* Unlock mba before copying segments */
if (!qproc->dump_mba_loaded) if (!qproc->dump_mba_loaded) {
ret = q6v5_mba_load(qproc); ret = q6v5_reload_mba(rproc);
if (!ret) {
/* Reset ownership back to Linux to copy segments */
ret = q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm,
true, false,
qproc->mpss_phys,
qproc->mpss_size);
}
}
if (!ptr || ret) if (!ptr || ret)
memset(dest, 0xff, segment->size); memset(dest, 0xff, segment->size);
...@@ -1198,8 +1259,14 @@ static void qcom_q6v5_dump_segment(struct rproc *rproc, ...@@ -1198,8 +1259,14 @@ static void qcom_q6v5_dump_segment(struct rproc *rproc,
/* Reclaim mba after copying segments */ /* Reclaim mba after copying segments */
if (qproc->dump_segment_mask == qproc->dump_complete_mask) { if (qproc->dump_segment_mask == qproc->dump_complete_mask) {
if (qproc->dump_mba_loaded) if (qproc->dump_mba_loaded) {
/* Try to reset ownership back to Q6 */
q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm,
false, true,
qproc->mpss_phys,
qproc->mpss_size);
q6v5_mba_reclaim(qproc); q6v5_mba_reclaim(qproc);
}
} }
} }
...@@ -1225,8 +1292,8 @@ static int q6v5_start(struct rproc *rproc) ...@@ -1225,8 +1292,8 @@ static int q6v5_start(struct rproc *rproc)
goto reclaim_mpss; goto reclaim_mpss;
} }
xfermemop_ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, false, xfermemop_ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, true,
qproc->mba_phys, false, qproc->mba_phys,
qproc->mba_size); qproc->mba_size);
if (xfermemop_ret) if (xfermemop_ret)
dev_err(qproc->dev, dev_err(qproc->dev,
...@@ -1239,10 +1306,6 @@ static int q6v5_start(struct rproc *rproc) ...@@ -1239,10 +1306,6 @@ static int q6v5_start(struct rproc *rproc)
return 0; return 0;
reclaim_mpss: reclaim_mpss:
xfermemop_ret = q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm,
false, qproc->mpss_phys,
qproc->mpss_size);
WARN_ON(xfermemop_ret);
q6v5_mba_reclaim(qproc); q6v5_mba_reclaim(qproc);
return ret; return ret;
...@@ -1264,7 +1327,7 @@ static int q6v5_stop(struct rproc *rproc) ...@@ -1264,7 +1327,7 @@ static int q6v5_stop(struct rproc *rproc)
return 0; return 0;
} }
static void *q6v5_da_to_va(struct rproc *rproc, u64 da, int len) static void *q6v5_da_to_va(struct rproc *rproc, u64 da, size_t len)
{ {
struct q6v5 *qproc = rproc->priv; struct q6v5 *qproc = rproc->priv;
int offset; int offset;
......
...@@ -222,7 +222,7 @@ static int adsp_stop(struct rproc *rproc) ...@@ -222,7 +222,7 @@ static int adsp_stop(struct rproc *rproc)
return ret; return ret;
} }
static void *adsp_da_to_va(struct rproc *rproc, u64 da, int len) static void *adsp_da_to_va(struct rproc *rproc, u64 da, size_t len)
{ {
struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv; struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
int offset; int offset;
...@@ -234,12 +234,20 @@ static void *adsp_da_to_va(struct rproc *rproc, u64 da, int len) ...@@ -234,12 +234,20 @@ static void *adsp_da_to_va(struct rproc *rproc, u64 da, int len)
return adsp->mem_region + offset; return adsp->mem_region + offset;
} }
static unsigned long adsp_panic(struct rproc *rproc)
{
struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
return qcom_q6v5_panic(&adsp->q6v5);
}
static const struct rproc_ops adsp_ops = { static const struct rproc_ops adsp_ops = {
.start = adsp_start, .start = adsp_start,
.stop = adsp_stop, .stop = adsp_stop,
.da_to_va = adsp_da_to_va, .da_to_va = adsp_da_to_va,
.parse_fw = qcom_register_dump_segments, .parse_fw = qcom_register_dump_segments,
.load = adsp_load, .load = adsp_load,
.panic = adsp_panic,
}; };
static int adsp_init_clock(struct qcom_adsp *adsp) static int adsp_init_clock(struct qcom_adsp *adsp)
......
...@@ -406,7 +406,7 @@ static int q6v5_wcss_stop(struct rproc *rproc) ...@@ -406,7 +406,7 @@ static int q6v5_wcss_stop(struct rproc *rproc)
return 0; return 0;
} }
static void *q6v5_wcss_da_to_va(struct rproc *rproc, u64 da, int len) static void *q6v5_wcss_da_to_va(struct rproc *rproc, u64 da, size_t len)
{ {
struct q6v5_wcss *wcss = rproc->priv; struct q6v5_wcss *wcss = rproc->priv;
int offset; int offset;
......
...@@ -287,7 +287,7 @@ static int wcnss_stop(struct rproc *rproc) ...@@ -287,7 +287,7 @@ static int wcnss_stop(struct rproc *rproc)
return ret; return ret;
} }
static void *wcnss_da_to_va(struct rproc *rproc, u64 da, int len) static void *wcnss_da_to_va(struct rproc *rproc, u64 da, size_t len)
{ {
struct qcom_wcnss *wcnss = (struct qcom_wcnss *)rproc->priv; struct qcom_wcnss *wcnss = (struct qcom_wcnss *)rproc->priv;
int offset; int offset;
......
This diff is collapsed.
...@@ -138,16 +138,16 @@ rproc_recovery_write(struct file *filp, const char __user *user_buf, ...@@ -138,16 +138,16 @@ rproc_recovery_write(struct file *filp, const char __user *user_buf,
buf[count - 1] = '\0'; buf[count - 1] = '\0';
if (!strncmp(buf, "enabled", count)) { if (!strncmp(buf, "enabled", count)) {
/* change the flag and begin the recovery process if needed */
rproc->recovery_disabled = false; rproc->recovery_disabled = false;
/* if rproc has crashed, trigger recovery */ rproc_trigger_recovery(rproc);
if (rproc->state == RPROC_CRASHED)
rproc_trigger_recovery(rproc);
} else if (!strncmp(buf, "disabled", count)) { } else if (!strncmp(buf, "disabled", count)) {
rproc->recovery_disabled = true; rproc->recovery_disabled = true;
} else if (!strncmp(buf, "recover", count)) { } else if (!strncmp(buf, "recover", count)) {
/* if rproc has crashed, trigger recovery */ /* begin the recovery process without changing the flag */
if (rproc->state == RPROC_CRASHED) rproc_trigger_recovery(rproc);
rproc_trigger_recovery(rproc); } else {
return -EINVAL;
} }
return count; return count;
...@@ -293,7 +293,7 @@ static int rproc_carveouts_show(struct seq_file *seq, void *p) ...@@ -293,7 +293,7 @@ static int rproc_carveouts_show(struct seq_file *seq, void *p)
seq_printf(seq, "\tVirtual address: %pK\n", carveout->va); seq_printf(seq, "\tVirtual address: %pK\n", carveout->va);
seq_printf(seq, "\tDMA address: %pad\n", &carveout->dma); seq_printf(seq, "\tDMA address: %pad\n", &carveout->dma);
seq_printf(seq, "\tDevice address: 0x%x\n", carveout->da); seq_printf(seq, "\tDevice address: 0x%x\n", carveout->da);
seq_printf(seq, "\tLength: 0x%x Bytes\n\n", carveout->len); seq_printf(seq, "\tLength: 0x%zx Bytes\n\n", carveout->len);
} }
return 0; return 0;
...@@ -349,7 +349,7 @@ void rproc_create_debug_dir(struct rproc *rproc) ...@@ -349,7 +349,7 @@ void rproc_create_debug_dir(struct rproc *rproc)
debugfs_create_file("name", 0400, rproc->dbg_dir, debugfs_create_file("name", 0400, rproc->dbg_dir,
rproc, &rproc_name_ops); rproc, &rproc_name_ops);
debugfs_create_file("recovery", 0400, rproc->dbg_dir, debugfs_create_file("recovery", 0600, rproc->dbg_dir,
rproc, &rproc_recovery_ops); rproc, &rproc_recovery_ops);
debugfs_create_file("crash", 0200, rproc->dbg_dir, debugfs_create_file("crash", 0200, rproc->dbg_dir,
rproc, &rproc_crash_ops); rproc, &rproc_crash_ops);
......
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Remote processor elf helpers defines
*
* Copyright (C) 2020 Kalray, Inc.
*/
#ifndef REMOTEPROC_ELF_LOADER_H
#define REMOTEPROC_ELF_LOADER_H
#include <linux/elf.h>
#include <linux/types.h>
/**
* fw_elf_get_class - Get elf class
* @fw: the ELF firmware image
*
* Note that we use and elf32_hdr to access the class since the start of the
* struct is the same for both elf class
*
* Return: elf class of the firmware
*/
static inline u8 fw_elf_get_class(const struct firmware *fw)
{
struct elf32_hdr *ehdr = (struct elf32_hdr *)fw->data;
return ehdr->e_ident[EI_CLASS];
}
static inline void elf_hdr_init_ident(struct elf32_hdr *hdr, u8 class)
{
memcpy(hdr->e_ident, ELFMAG, SELFMAG);
hdr->e_ident[EI_CLASS] = class;
hdr->e_ident[EI_DATA] = ELFDATA2LSB;
hdr->e_ident[EI_VERSION] = EV_CURRENT;
hdr->e_ident[EI_OSABI] = ELFOSABI_NONE;
}
/* Generate getter and setter for a specific elf struct/field */
#define ELF_GEN_FIELD_GET_SET(__s, __field, __type) \
static inline __type elf_##__s##_get_##__field(u8 class, const void *arg) \
{ \
if (class == ELFCLASS32) \
return (__type) ((const struct elf32_##__s *) arg)->__field; \
else \
return (__type) ((const struct elf64_##__s *) arg)->__field; \
} \
static inline void elf_##__s##_set_##__field(u8 class, void *arg, \
__type value) \
{ \
if (class == ELFCLASS32) \
((struct elf32_##__s *) arg)->__field = (__type) value; \
else \
((struct elf64_##__s *) arg)->__field = (__type) value; \
}
ELF_GEN_FIELD_GET_SET(hdr, e_entry, u64)
ELF_GEN_FIELD_GET_SET(hdr, e_phnum, u16)
ELF_GEN_FIELD_GET_SET(hdr, e_shnum, u16)
ELF_GEN_FIELD_GET_SET(hdr, e_phoff, u64)
ELF_GEN_FIELD_GET_SET(hdr, e_shoff, u64)
ELF_GEN_FIELD_GET_SET(hdr, e_shstrndx, u16)
ELF_GEN_FIELD_GET_SET(hdr, e_machine, u16)
ELF_GEN_FIELD_GET_SET(hdr, e_type, u16)
ELF_GEN_FIELD_GET_SET(hdr, e_version, u32)
ELF_GEN_FIELD_GET_SET(hdr, e_ehsize, u32)
ELF_GEN_FIELD_GET_SET(hdr, e_phentsize, u16)
ELF_GEN_FIELD_GET_SET(phdr, p_paddr, u64)
ELF_GEN_FIELD_GET_SET(phdr, p_vaddr, u64)
ELF_GEN_FIELD_GET_SET(phdr, p_filesz, u64)
ELF_GEN_FIELD_GET_SET(phdr, p_memsz, u64)
ELF_GEN_FIELD_GET_SET(phdr, p_type, u32)
ELF_GEN_FIELD_GET_SET(phdr, p_offset, u64)
ELF_GEN_FIELD_GET_SET(phdr, p_flags, u32)
ELF_GEN_FIELD_GET_SET(phdr, p_align, u64)
ELF_GEN_FIELD_GET_SET(shdr, sh_size, u64)
ELF_GEN_FIELD_GET_SET(shdr, sh_offset, u64)
ELF_GEN_FIELD_GET_SET(shdr, sh_name, u32)
ELF_GEN_FIELD_GET_SET(shdr, sh_addr, u64)
#define ELF_STRUCT_SIZE(__s) \
static inline unsigned long elf_size_of_##__s(u8 class) \
{ \
if (class == ELFCLASS32)\
return sizeof(struct elf32_##__s); \
else \
return sizeof(struct elf64_##__s); \
}
ELF_STRUCT_SIZE(shdr)
ELF_STRUCT_SIZE(phdr)
ELF_STRUCT_SIZE(hdr)
#endif /* REMOTEPROC_ELF_LOADER_H */
...@@ -23,20 +23,29 @@ ...@@ -23,20 +23,29 @@
#include <linux/elf.h> #include <linux/elf.h>
#include "remoteproc_internal.h" #include "remoteproc_internal.h"
#include "remoteproc_elf_helpers.h"
/** /**
* rproc_elf_sanity_check() - Sanity Check ELF firmware image * rproc_elf_sanity_check() - Sanity Check for ELF32/ELF64 firmware image
* @rproc: the remote processor handle * @rproc: the remote processor handle
* @fw: the ELF firmware image * @fw: the ELF firmware image
* *
* Make sure this fw image is sane. * Make sure this fw image is sane (ie a correct ELF32/ELF64 file).
*/ */
int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw) int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw)
{ {
const char *name = rproc->firmware; const char *name = rproc->firmware;
struct device *dev = &rproc->dev; struct device *dev = &rproc->dev;
/*
* Elf files are beginning with the same structure. Thus, to simplify
* header parsing, we can use the elf32_hdr one for both elf64 and
* elf32.
*/
struct elf32_hdr *ehdr; struct elf32_hdr *ehdr;
u32 elf_shdr_get_size;
u64 phoff, shoff;
char class; char class;
u16 phnum;
if (!fw) { if (!fw) {
dev_err(dev, "failed to load %s\n", name); dev_err(dev, "failed to load %s\n", name);
...@@ -50,13 +59,22 @@ int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw) ...@@ -50,13 +59,22 @@ int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw)
ehdr = (struct elf32_hdr *)fw->data; ehdr = (struct elf32_hdr *)fw->data;
/* We only support ELF32 at this point */ if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG)) {
dev_err(dev, "Image is corrupted (bad magic)\n");
return -EINVAL;
}
class = ehdr->e_ident[EI_CLASS]; class = ehdr->e_ident[EI_CLASS];
if (class != ELFCLASS32) { if (class != ELFCLASS32 && class != ELFCLASS64) {
dev_err(dev, "Unsupported class: %d\n", class); dev_err(dev, "Unsupported class: %d\n", class);
return -EINVAL; return -EINVAL;
} }
if (class == ELFCLASS64 && fw->size < sizeof(struct elf64_hdr)) {
dev_err(dev, "elf64 header is too small\n");
return -EINVAL;
}
/* We assume the firmware has the same endianness as the host */ /* We assume the firmware has the same endianness as the host */
# ifdef __LITTLE_ENDIAN # ifdef __LITTLE_ENDIAN
if (ehdr->e_ident[EI_DATA] != ELFDATA2LSB) { if (ehdr->e_ident[EI_DATA] != ELFDATA2LSB) {
...@@ -67,30 +85,54 @@ int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw) ...@@ -67,30 +85,54 @@ int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw)
return -EINVAL; return -EINVAL;
} }
if (fw->size < ehdr->e_shoff + sizeof(struct elf32_shdr)) { phoff = elf_hdr_get_e_phoff(class, fw->data);
dev_err(dev, "Image is too small\n"); shoff = elf_hdr_get_e_shoff(class, fw->data);
return -EINVAL; phnum = elf_hdr_get_e_phnum(class, fw->data);
} elf_shdr_get_size = elf_size_of_shdr(class);
if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG)) { if (fw->size < shoff + elf_shdr_get_size) {
dev_err(dev, "Image is corrupted (bad magic)\n"); dev_err(dev, "Image is too small\n");
return -EINVAL; return -EINVAL;
} }
if (ehdr->e_phnum == 0) { if (phnum == 0) {
dev_err(dev, "No loadable segments\n"); dev_err(dev, "No loadable segments\n");
return -EINVAL; return -EINVAL;
} }
if (ehdr->e_phoff > fw->size) { if (phoff > fw->size) {
dev_err(dev, "Firmware size is too small\n"); dev_err(dev, "Firmware size is too small\n");
return -EINVAL; return -EINVAL;
} }
dev_dbg(dev, "Firmware is an elf%d file\n",
class == ELFCLASS32 ? 32 : 64);
return 0; return 0;
} }
EXPORT_SYMBOL(rproc_elf_sanity_check); EXPORT_SYMBOL(rproc_elf_sanity_check);
/**
* rproc_elf_sanity_check() - Sanity Check ELF32 firmware image
* @rproc: the remote processor handle
* @fw: the ELF32 firmware image
*
* Make sure this fw image is sane.
*/
int rproc_elf32_sanity_check(struct rproc *rproc, const struct firmware *fw)
{
int ret = rproc_elf_sanity_check(rproc, fw);
if (ret)
return ret;
if (fw_elf_get_class(fw) == ELFCLASS32)
return 0;
return -EINVAL;
}
EXPORT_SYMBOL(rproc_elf32_sanity_check);
/** /**
* rproc_elf_get_boot_addr() - Get rproc's boot address. * rproc_elf_get_boot_addr() - Get rproc's boot address.
* @rproc: the remote processor handle * @rproc: the remote processor handle
...@@ -102,11 +144,9 @@ EXPORT_SYMBOL(rproc_elf_sanity_check); ...@@ -102,11 +144,9 @@ EXPORT_SYMBOL(rproc_elf_sanity_check);
* Note that the boot address is not a configurable property of all remote * Note that the boot address is not a configurable property of all remote
* processors. Some will always boot at a specific hard-coded address. * processors. Some will always boot at a specific hard-coded address.
*/ */
u32 rproc_elf_get_boot_addr(struct rproc *rproc, const struct firmware *fw) u64 rproc_elf_get_boot_addr(struct rproc *rproc, const struct firmware *fw)
{ {
struct elf32_hdr *ehdr = (struct elf32_hdr *)fw->data; return elf_hdr_get_e_entry(fw_elf_get_class(fw), fw->data);
return ehdr->e_entry;
} }
EXPORT_SYMBOL(rproc_elf_get_boot_addr); EXPORT_SYMBOL(rproc_elf_get_boot_addr);
...@@ -137,53 +177,65 @@ EXPORT_SYMBOL(rproc_elf_get_boot_addr); ...@@ -137,53 +177,65 @@ EXPORT_SYMBOL(rproc_elf_get_boot_addr);
int rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw) int rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw)
{ {
struct device *dev = &rproc->dev; struct device *dev = &rproc->dev;
struct elf32_hdr *ehdr; const void *ehdr, *phdr;
struct elf32_phdr *phdr;
int i, ret = 0; int i, ret = 0;
u16 phnum;
const u8 *elf_data = fw->data; const u8 *elf_data = fw->data;
u8 class = fw_elf_get_class(fw);
u32 elf_phdr_get_size = elf_size_of_phdr(class);
ehdr = (struct elf32_hdr *)elf_data; ehdr = elf_data;
phdr = (struct elf32_phdr *)(elf_data + ehdr->e_phoff); phnum = elf_hdr_get_e_phnum(class, ehdr);
phdr = elf_data + elf_hdr_get_e_phoff(class, ehdr);
/* go through the available ELF segments */ /* go through the available ELF segments */
for (i = 0; i < ehdr->e_phnum; i++, phdr++) { for (i = 0; i < phnum; i++, phdr += elf_phdr_get_size) {
u32 da = phdr->p_paddr; u64 da = elf_phdr_get_p_paddr(class, phdr);
u32 memsz = phdr->p_memsz; u64 memsz = elf_phdr_get_p_memsz(class, phdr);
u32 filesz = phdr->p_filesz; u64 filesz = elf_phdr_get_p_filesz(class, phdr);
u32 offset = phdr->p_offset; u64 offset = elf_phdr_get_p_offset(class, phdr);
u32 type = elf_phdr_get_p_type(class, phdr);
void *ptr; void *ptr;
if (phdr->p_type != PT_LOAD) if (type != PT_LOAD)
continue; continue;
dev_dbg(dev, "phdr: type %d da 0x%x memsz 0x%x filesz 0x%x\n", dev_dbg(dev, "phdr: type %d da 0x%llx memsz 0x%llx filesz 0x%llx\n",
phdr->p_type, da, memsz, filesz); type, da, memsz, filesz);
if (filesz > memsz) { if (filesz > memsz) {
dev_err(dev, "bad phdr filesz 0x%x memsz 0x%x\n", dev_err(dev, "bad phdr filesz 0x%llx memsz 0x%llx\n",
filesz, memsz); filesz, memsz);
ret = -EINVAL; ret = -EINVAL;
break; break;
} }
if (offset + filesz > fw->size) { if (offset + filesz > fw->size) {
dev_err(dev, "truncated fw: need 0x%x avail 0x%zx\n", dev_err(dev, "truncated fw: need 0x%llx avail 0x%zx\n",
offset + filesz, fw->size); offset + filesz, fw->size);
ret = -EINVAL; ret = -EINVAL;
break; break;
} }
if (!rproc_u64_fit_in_size_t(memsz)) {
dev_err(dev, "size (%llx) does not fit in size_t type\n",
memsz);
ret = -EOVERFLOW;
break;
}
/* grab the kernel address for this device address */ /* grab the kernel address for this device address */
ptr = rproc_da_to_va(rproc, da, memsz); ptr = rproc_da_to_va(rproc, da, memsz);
if (!ptr) { if (!ptr) {
dev_err(dev, "bad phdr da 0x%x mem 0x%x\n", da, memsz); dev_err(dev, "bad phdr da 0x%llx mem 0x%llx\n", da,
memsz);
ret = -EINVAL; ret = -EINVAL;
break; break;
} }
/* put the segment where the remote processor expects it */ /* put the segment where the remote processor expects it */
if (phdr->p_filesz) if (filesz)
memcpy(ptr, elf_data + phdr->p_offset, filesz); memcpy(ptr, elf_data + offset, filesz);
/* /*
* Zero out remaining memory for this segment. * Zero out remaining memory for this segment.
...@@ -196,28 +248,42 @@ int rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw) ...@@ -196,28 +248,42 @@ int rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw)
memset(ptr + filesz, 0, memsz - filesz); memset(ptr + filesz, 0, memsz - filesz);
} }
if (ret == 0)
rproc->elf_class = class;
return ret; return ret;
} }
EXPORT_SYMBOL(rproc_elf_load_segments); EXPORT_SYMBOL(rproc_elf_load_segments);
static struct elf32_shdr * static const void *
find_table(struct device *dev, struct elf32_hdr *ehdr, size_t fw_size) find_table(struct device *dev, const struct firmware *fw)
{ {
struct elf32_shdr *shdr; const void *shdr, *name_table_shdr;
int i; int i;
const char *name_table; const char *name_table;
struct resource_table *table = NULL; struct resource_table *table = NULL;
const u8 *elf_data = (void *)ehdr; const u8 *elf_data = (void *)fw->data;
u8 class = fw_elf_get_class(fw);
size_t fw_size = fw->size;
const void *ehdr = elf_data;
u16 shnum = elf_hdr_get_e_shnum(class, ehdr);
u32 elf_shdr_get_size = elf_size_of_shdr(class);
u16 shstrndx = elf_hdr_get_e_shstrndx(class, ehdr);
/* look for the resource table and handle it */ /* look for the resource table and handle it */
shdr = (struct elf32_shdr *)(elf_data + ehdr->e_shoff); /* First, get the section header according to the elf class */
name_table = elf_data + shdr[ehdr->e_shstrndx].sh_offset; shdr = elf_data + elf_hdr_get_e_shoff(class, ehdr);
/* Compute name table section header entry in shdr array */
for (i = 0; i < ehdr->e_shnum; i++, shdr++) { name_table_shdr = shdr + (shstrndx * elf_shdr_get_size);
u32 size = shdr->sh_size; /* Finally, compute the name table section address in elf */
u32 offset = shdr->sh_offset; name_table = elf_data + elf_shdr_get_sh_offset(class, name_table_shdr);
if (strcmp(name_table + shdr->sh_name, ".resource_table")) for (i = 0; i < shnum; i++, shdr += elf_shdr_get_size) {
u64 size = elf_shdr_get_sh_size(class, shdr);
u64 offset = elf_shdr_get_sh_offset(class, shdr);
u32 name = elf_shdr_get_sh_name(class, shdr);
if (strcmp(name_table + name, ".resource_table"))
continue; continue;
table = (struct resource_table *)(elf_data + offset); table = (struct resource_table *)(elf_data + offset);
...@@ -270,21 +336,21 @@ find_table(struct device *dev, struct elf32_hdr *ehdr, size_t fw_size) ...@@ -270,21 +336,21 @@ find_table(struct device *dev, struct elf32_hdr *ehdr, size_t fw_size)
*/ */
int rproc_elf_load_rsc_table(struct rproc *rproc, const struct firmware *fw) int rproc_elf_load_rsc_table(struct rproc *rproc, const struct firmware *fw)
{ {
struct elf32_hdr *ehdr; const void *shdr;
struct elf32_shdr *shdr;
struct device *dev = &rproc->dev; struct device *dev = &rproc->dev;
struct resource_table *table = NULL; struct resource_table *table = NULL;
const u8 *elf_data = fw->data; const u8 *elf_data = fw->data;
size_t tablesz; size_t tablesz;
u8 class = fw_elf_get_class(fw);
u64 sh_offset;
ehdr = (struct elf32_hdr *)elf_data; shdr = find_table(dev, fw);
shdr = find_table(dev, ehdr, fw->size);
if (!shdr) if (!shdr)
return -EINVAL; return -EINVAL;
table = (struct resource_table *)(elf_data + shdr->sh_offset); sh_offset = elf_shdr_get_sh_offset(class, shdr);
tablesz = shdr->sh_size; table = (struct resource_table *)(elf_data + sh_offset);
tablesz = elf_shdr_get_sh_size(class, shdr);
/* /*
* Create a copy of the resource table. When a virtio device starts * Create a copy of the resource table. When a virtio device starts
...@@ -317,13 +383,24 @@ EXPORT_SYMBOL(rproc_elf_load_rsc_table); ...@@ -317,13 +383,24 @@ EXPORT_SYMBOL(rproc_elf_load_rsc_table);
struct resource_table *rproc_elf_find_loaded_rsc_table(struct rproc *rproc, struct resource_table *rproc_elf_find_loaded_rsc_table(struct rproc *rproc,
const struct firmware *fw) const struct firmware *fw)
{ {
struct elf32_hdr *ehdr = (struct elf32_hdr *)fw->data; const void *shdr;
struct elf32_shdr *shdr; u64 sh_addr, sh_size;
u8 class = fw_elf_get_class(fw);
struct device *dev = &rproc->dev;
shdr = find_table(&rproc->dev, ehdr, fw->size); shdr = find_table(&rproc->dev, fw);
if (!shdr) if (!shdr)
return NULL; return NULL;
return rproc_da_to_va(rproc, shdr->sh_addr, shdr->sh_size); sh_addr = elf_shdr_get_sh_addr(class, shdr);
sh_size = elf_shdr_get_sh_size(class, shdr);
if (!rproc_u64_fit_in_size_t(sh_size)) {
dev_err(dev, "size (%llx) does not fit in size_t type\n",
sh_size);
return NULL;
}
return rproc_da_to_va(rproc, sh_addr, sh_size);
} }
EXPORT_SYMBOL(rproc_elf_find_loaded_rsc_table); EXPORT_SYMBOL(rproc_elf_find_loaded_rsc_table);
...@@ -50,12 +50,13 @@ void rproc_exit_sysfs(void); ...@@ -50,12 +50,13 @@ void rproc_exit_sysfs(void);
void rproc_free_vring(struct rproc_vring *rvring); void rproc_free_vring(struct rproc_vring *rvring);
int rproc_alloc_vring(struct rproc_vdev *rvdev, int i); int rproc_alloc_vring(struct rproc_vdev *rvdev, int i);
void *rproc_da_to_va(struct rproc *rproc, u64 da, int len); void *rproc_da_to_va(struct rproc *rproc, u64 da, size_t len);
phys_addr_t rproc_va_to_pa(void *cpu_addr); phys_addr_t rproc_va_to_pa(void *cpu_addr);
int rproc_trigger_recovery(struct rproc *rproc); int rproc_trigger_recovery(struct rproc *rproc);
int rproc_elf32_sanity_check(struct rproc *rproc, const struct firmware *fw);
int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw); int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw);
u32 rproc_elf_get_boot_addr(struct rproc *rproc, const struct firmware *fw); u64 rproc_elf_get_boot_addr(struct rproc *rproc, const struct firmware *fw);
int rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw); int rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw);
int rproc_elf_load_rsc_table(struct rproc *rproc, const struct firmware *fw); int rproc_elf_load_rsc_table(struct rproc *rproc, const struct firmware *fw);
struct resource_table *rproc_elf_find_loaded_rsc_table(struct rproc *rproc, struct resource_table *rproc_elf_find_loaded_rsc_table(struct rproc *rproc,
...@@ -73,7 +74,7 @@ int rproc_fw_sanity_check(struct rproc *rproc, const struct firmware *fw) ...@@ -73,7 +74,7 @@ int rproc_fw_sanity_check(struct rproc *rproc, const struct firmware *fw)
} }
static inline static inline
u32 rproc_get_boot_addr(struct rproc *rproc, const struct firmware *fw) u64 rproc_get_boot_addr(struct rproc *rproc, const struct firmware *fw)
{ {
if (rproc->ops->get_boot_addr) if (rproc->ops->get_boot_addr)
return rproc->ops->get_boot_addr(rproc, fw); return rproc->ops->get_boot_addr(rproc, fw);
...@@ -119,4 +120,13 @@ struct resource_table *rproc_find_loaded_rsc_table(struct rproc *rproc, ...@@ -119,4 +120,13 @@ struct resource_table *rproc_find_loaded_rsc_table(struct rproc *rproc,
return NULL; return NULL;
} }
static inline
bool rproc_u64_fit_in_size_t(u64 val)
{
if (sizeof(size_t) == sizeof(u64))
return true;
return (val <= (size_t) -1);
}
#endif /* REMOTEPROC_INTERNAL_H */ #endif /* REMOTEPROC_INTERNAL_H */
...@@ -320,6 +320,7 @@ static void rproc_virtio_dev_release(struct device *dev) ...@@ -320,6 +320,7 @@ static void rproc_virtio_dev_release(struct device *dev)
/** /**
* rproc_add_virtio_dev() - register an rproc-induced virtio device * rproc_add_virtio_dev() - register an rproc-induced virtio device
* @rvdev: the remote vdev * @rvdev: the remote vdev
* @id: the device type identification (used to match it with a driver).
* *
* This function registers a virtio device. This vdev's partent is * This function registers a virtio device. This vdev's partent is
* the rproc device. * the rproc device.
...@@ -334,6 +335,13 @@ int rproc_add_virtio_dev(struct rproc_vdev *rvdev, int id) ...@@ -334,6 +335,13 @@ int rproc_add_virtio_dev(struct rproc_vdev *rvdev, int id)
struct rproc_mem_entry *mem; struct rproc_mem_entry *mem;
int ret; int ret;
if (rproc->ops->kick == NULL) {
ret = -EINVAL;
dev_err(dev, ".kick method not defined for %s",
rproc->name);
goto out;
}
/* Try to find dedicated vdev buffer carveout */ /* Try to find dedicated vdev buffer carveout */
mem = rproc_find_carveout_by_name(rproc, "vdev%dbuffer", rvdev->index); mem = rproc_find_carveout_by_name(rproc, "vdev%dbuffer", rvdev->index);
if (mem) { if (mem) {
......
...@@ -190,7 +190,7 @@ static int st_rproc_start(struct rproc *rproc) ...@@ -190,7 +190,7 @@ static int st_rproc_start(struct rproc *rproc)
} }
} }
dev_info(&rproc->dev, "Started from 0x%x\n", rproc->bootaddr); dev_info(&rproc->dev, "Started from 0x%llx\n", rproc->bootaddr);
return 0; return 0;
...@@ -233,7 +233,7 @@ static const struct rproc_ops st_rproc_ops = { ...@@ -233,7 +233,7 @@ static const struct rproc_ops st_rproc_ops = {
.parse_fw = st_rproc_parse_fw, .parse_fw = st_rproc_parse_fw,
.load = rproc_elf_load_segments, .load = rproc_elf_load_segments,
.find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table, .find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table,
.sanity_check = rproc_elf_sanity_check, .sanity_check = rproc_elf32_sanity_check,
.get_boot_addr = rproc_elf_get_boot_addr, .get_boot_addr = rproc_elf_get_boot_addr,
}; };
......
...@@ -174,7 +174,7 @@ static int slim_rproc_stop(struct rproc *rproc) ...@@ -174,7 +174,7 @@ static int slim_rproc_stop(struct rproc *rproc)
return 0; return 0;
} }
static void *slim_rproc_da_to_va(struct rproc *rproc, u64 da, int len) static void *slim_rproc_da_to_va(struct rproc *rproc, u64 da, size_t len)
{ {
struct st_slim_rproc *slim_rproc = rproc->priv; struct st_slim_rproc *slim_rproc = rproc->priv;
void *va = NULL; void *va = NULL;
...@@ -191,7 +191,7 @@ static void *slim_rproc_da_to_va(struct rproc *rproc, u64 da, int len) ...@@ -191,7 +191,7 @@ static void *slim_rproc_da_to_va(struct rproc *rproc, u64 da, int len)
} }
} }
dev_dbg(&rproc->dev, "da = 0x%llx len = 0x%x va = 0x%pK\n", dev_dbg(&rproc->dev, "da = 0x%llx len = 0x%zx va = 0x%pK\n",
da, len, va); da, len, va);
return va; return va;
...@@ -203,7 +203,7 @@ static const struct rproc_ops slim_rproc_ops = { ...@@ -203,7 +203,7 @@ static const struct rproc_ops slim_rproc_ops = {
.da_to_va = slim_rproc_da_to_va, .da_to_va = slim_rproc_da_to_va,
.get_boot_addr = rproc_elf_get_boot_addr, .get_boot_addr = rproc_elf_get_boot_addr,
.load = rproc_elf_load_segments, .load = rproc_elf_load_segments,
.sanity_check = rproc_elf_sanity_check, .sanity_check = rproc_elf32_sanity_check,
}; };
/** /**
......
...@@ -505,7 +505,7 @@ static struct rproc_ops st_rproc_ops = { ...@@ -505,7 +505,7 @@ static struct rproc_ops st_rproc_ops = {
.load = rproc_elf_load_segments, .load = rproc_elf_load_segments,
.parse_fw = stm32_rproc_parse_fw, .parse_fw = stm32_rproc_parse_fw,
.find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table, .find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table,
.sanity_check = rproc_elf_sanity_check, .sanity_check = rproc_elf32_sanity_check,
.get_boot_addr = rproc_elf_get_boot_addr, .get_boot_addr = rproc_elf_get_boot_addr,
}; };
...@@ -602,7 +602,7 @@ static int stm32_rproc_parse_dt(struct platform_device *pdev) ...@@ -602,7 +602,7 @@ static int stm32_rproc_parse_dt(struct platform_device *pdev)
err = stm32_rproc_get_syscon(np, "st,syscfg-pdds", &ddata->pdds); err = stm32_rproc_get_syscon(np, "st,syscfg-pdds", &ddata->pdds);
if (err) if (err)
dev_warn(dev, "failed to get pdds\n"); dev_info(dev, "failed to get pdds\n");
rproc->auto_boot = of_property_read_bool(np, "st,auto-boot"); rproc->auto_boot = of_property_read_bool(np, "st,auto-boot");
......
...@@ -80,14 +80,14 @@ static int wkup_m3_rproc_stop(struct rproc *rproc) ...@@ -80,14 +80,14 @@ static int wkup_m3_rproc_stop(struct rproc *rproc)
return 0; return 0;
} }
static void *wkup_m3_rproc_da_to_va(struct rproc *rproc, u64 da, int len) static void *wkup_m3_rproc_da_to_va(struct rproc *rproc, u64 da, size_t len)
{ {
struct wkup_m3_rproc *wkupm3 = rproc->priv; struct wkup_m3_rproc *wkupm3 = rproc->priv;
void *va = NULL; void *va = NULL;
int i; int i;
u32 offset; u32 offset;
if (len <= 0) if (len == 0)
return NULL; return NULL;
for (i = 0; i < WKUPM3_MEM_MAX; i++) { for (i = 0; i < WKUPM3_MEM_MAX; i++) {
......
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Remote Processor - omap-specific bits
*
* Copyright (C) 2011 Texas Instruments, Inc.
* Copyright (C) 2011 Google, Inc.
*/
#ifndef _PLAT_REMOTEPROC_H
#define _PLAT_REMOTEPROC_H
struct rproc_ops;
struct platform_device;
/*
* struct omap_rproc_pdata - omap remoteproc's platform data
* @name: the remoteproc's name
* @oh_name: omap hwmod device
* @oh_name_opt: optional, secondary omap hwmod device
* @firmware: name of firmware file to load
* @mbox_name: name of omap mailbox device to use with this rproc
* @ops: start/stop rproc handlers
* @device_enable: omap-specific handler for enabling a device
* @device_shutdown: omap-specific handler for shutting down a device
* @set_bootaddr: omap-specific handler for setting the rproc boot address
*/
struct omap_rproc_pdata {
const char *name;
const char *oh_name;
const char *oh_name_opt;
const char *firmware;
const char *mbox_name;
const struct rproc_ops *ops;
int (*device_enable)(struct platform_device *pdev);
int (*device_shutdown)(struct platform_device *pdev);
void (*set_bootaddr)(u32);
};
#if defined(CONFIG_OMAP_REMOTEPROC) || defined(CONFIG_OMAP_REMOTEPROC_MODULE)
void __init omap_rproc_reserve_cma(void);
#else
static inline void __init omap_rproc_reserve_cma(void)
{
}
#endif
#endif /* _PLAT_REMOTEPROC_H */
...@@ -329,7 +329,7 @@ struct rproc; ...@@ -329,7 +329,7 @@ struct rproc;
struct rproc_mem_entry { struct rproc_mem_entry {
void *va; void *va;
dma_addr_t dma; dma_addr_t dma;
int len; size_t len;
u32 da; u32 da;
void *priv; void *priv;
char name[32]; char name[32];
...@@ -369,12 +369,14 @@ enum rsc_handling_status { ...@@ -369,12 +369,14 @@ enum rsc_handling_status {
* expects to find it * expects to find it
* @sanity_check: sanity check the fw image * @sanity_check: sanity check the fw image
* @get_boot_addr: get boot address to entry point specified in firmware * @get_boot_addr: get boot address to entry point specified in firmware
* @panic: optional callback to react to system panic, core will delay
* panic at least the returned number of milliseconds
*/ */
struct rproc_ops { struct rproc_ops {
int (*start)(struct rproc *rproc); int (*start)(struct rproc *rproc);
int (*stop)(struct rproc *rproc); int (*stop)(struct rproc *rproc);
void (*kick)(struct rproc *rproc, int vqid); void (*kick)(struct rproc *rproc, int vqid);
void * (*da_to_va)(struct rproc *rproc, u64 da, int len); void * (*da_to_va)(struct rproc *rproc, u64 da, size_t len);
int (*parse_fw)(struct rproc *rproc, const struct firmware *fw); int (*parse_fw)(struct rproc *rproc, const struct firmware *fw);
int (*handle_rsc)(struct rproc *rproc, u32 rsc_type, void *rsc, int (*handle_rsc)(struct rproc *rproc, u32 rsc_type, void *rsc,
int offset, int avail); int offset, int avail);
...@@ -382,7 +384,8 @@ struct rproc_ops { ...@@ -382,7 +384,8 @@ struct rproc_ops {
struct rproc *rproc, const struct firmware *fw); struct rproc *rproc, const struct firmware *fw);
int (*load)(struct rproc *rproc, const struct firmware *fw); int (*load)(struct rproc *rproc, const struct firmware *fw);
int (*sanity_check)(struct rproc *rproc, const struct firmware *fw); int (*sanity_check)(struct rproc *rproc, const struct firmware *fw);
u32 (*get_boot_addr)(struct rproc *rproc, const struct firmware *fw); u64 (*get_boot_addr)(struct rproc *rproc, const struct firmware *fw);
unsigned long (*panic)(struct rproc *rproc);
}; };
/** /**
...@@ -498,7 +501,7 @@ struct rproc { ...@@ -498,7 +501,7 @@ struct rproc {
int num_traces; int num_traces;
struct list_head carveouts; struct list_head carveouts;
struct list_head mappings; struct list_head mappings;
u32 bootaddr; u64 bootaddr;
struct list_head rvdevs; struct list_head rvdevs;
struct list_head subdevs; struct list_head subdevs;
struct idr notifyids; struct idr notifyids;
...@@ -514,6 +517,7 @@ struct rproc { ...@@ -514,6 +517,7 @@ struct rproc {
bool auto_boot; bool auto_boot;
struct list_head dump_segments; struct list_head dump_segments;
int nb_vdev; int nb_vdev;
u8 elf_class;
}; };
/** /**
...@@ -599,13 +603,13 @@ void rproc_add_carveout(struct rproc *rproc, struct rproc_mem_entry *mem); ...@@ -599,13 +603,13 @@ void rproc_add_carveout(struct rproc *rproc, struct rproc_mem_entry *mem);
struct rproc_mem_entry * struct rproc_mem_entry *
rproc_mem_entry_init(struct device *dev, rproc_mem_entry_init(struct device *dev,
void *va, dma_addr_t dma, int len, u32 da, void *va, dma_addr_t dma, size_t len, u32 da,
int (*alloc)(struct rproc *, struct rproc_mem_entry *), int (*alloc)(struct rproc *, struct rproc_mem_entry *),
int (*release)(struct rproc *, struct rproc_mem_entry *), int (*release)(struct rproc *, struct rproc_mem_entry *),
const char *name, ...); const char *name, ...);
struct rproc_mem_entry * struct rproc_mem_entry *
rproc_of_resm_mem_entry_init(struct device *dev, u32 of_resm_idx, int len, rproc_of_resm_mem_entry_init(struct device *dev, u32 of_resm_idx, size_t len,
u32 da, const char *name, ...); u32 da, const char *name, ...);
int rproc_boot(struct rproc *rproc); int rproc_boot(struct rproc *rproc);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment