Commit 09102704 authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost

Pull virtio updates from Michael Tsirkin:

 - virtio-mem: paravirtualized memory hotplug

 - support doorbell mapping for vdpa

 - config interrupt support in ifc

 - fixes all over the place

* tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost: (40 commits)
  vhost/test: fix up after API change
  virtio_mem: convert device block size into 64bit
  virtio-mem: drop unnecessary initialization
  ifcvf: implement config interrupt in IFCVF
  vhost: replace -1 with VHOST_FILE_UNBIND in ioctls
  vhost_vdpa: Support config interrupt in vdpa
  ifcvf: ignore continuous setting same status value
  virtio-mem: Don't rely on implicit compiler padding for requests
  virtio-mem: Try to unplug the complete online memory block first
  virtio-mem: Use -ETXTBSY as error code if the device is busy
  virtio-mem: Unplug subblocks right-to-left
  virtio-mem: Drop manual check for already present memory
  virtio-mem: Add parent resource for all added "System RAM"
  virtio-mem: Better retry handling
  virtio-mem: Offline and remove completely unplugged memory blocks
  mm/memory_hotplug: Introduce offline_and_remove_memory()
  virtio-mem: Allow to offline partially unplugged memory blocks
  mm: Allow to offline unmovable PageOffline() pages via MEM_GOING_OFFLINE
  virtio-mem: Paravirtualized memory hotunplug part 2
  virtio-mem: Paravirtualized memory hotunplug part 1
  ...
parents 84fc461d 044e4b09
...@@ -18114,9 +18114,18 @@ F: drivers/virtio/ ...@@ -18114,9 +18114,18 @@ F: drivers/virtio/
F: include/linux/vdpa.h F: include/linux/vdpa.h
F: include/linux/virtio*.h F: include/linux/virtio*.h
F: include/uapi/linux/virtio_*.h F: include/uapi/linux/virtio_*.h
F: mm/balloon_compaction.c
F: tools/virtio/ F: tools/virtio/
VIRTIO BALLOON
M: "Michael S. Tsirkin" <mst@redhat.com>
M: David Hildenbrand <david@redhat.com>
L: virtualization@lists.linux-foundation.org
S: Maintained
F: drivers/virtio/virtio_balloon.c
F: include/uapi/linux/virtio_balloon.h
F: include/linux/balloon_compaction.h
F: mm/balloon_compaction.c
VIRTIO CRYPTO DRIVER VIRTIO CRYPTO DRIVER
M: Gonglei <arei.gonglei@huawei.com> M: Gonglei <arei.gonglei@huawei.com>
L: virtualization@lists.linux-foundation.org L: virtualization@lists.linux-foundation.org
...@@ -18182,6 +18191,13 @@ S: Maintained ...@@ -18182,6 +18191,13 @@ S: Maintained
F: drivers/iommu/virtio-iommu.c F: drivers/iommu/virtio-iommu.c
F: include/uapi/linux/virtio_iommu.h F: include/uapi/linux/virtio_iommu.h
VIRTIO MEM DRIVER
M: David Hildenbrand <david@redhat.com>
L: virtualization@lists.linux-foundation.org
S: Maintained
F: drivers/virtio/virtio_mem.c
F: include/uapi/linux/virtio_mem.h
VIRTUAL BOX GUEST DEVICE DRIVER VIRTUAL BOX GUEST DEVICE DRIVER
M: Hans de Goede <hdegoede@redhat.com> M: Hans de Goede <hdegoede@redhat.com>
M: Arnd Bergmann <arnd@arndb.de> M: Arnd Bergmann <arnd@arndb.de>
......
...@@ -35,6 +35,7 @@ int pxm_to_node(int pxm) ...@@ -35,6 +35,7 @@ int pxm_to_node(int pxm)
return NUMA_NO_NODE; return NUMA_NO_NODE;
return pxm_to_node_map[pxm]; return pxm_to_node_map[pxm];
} }
EXPORT_SYMBOL(pxm_to_node);
int node_to_pxm(int node) int node_to_pxm(int node)
{ {
......
...@@ -350,13 +350,18 @@ __virtio_crypto_skcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req, ...@@ -350,13 +350,18 @@ __virtio_crypto_skcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req,
int err; int err;
unsigned long flags; unsigned long flags;
struct scatterlist outhdr, iv_sg, status_sg, **sgs; struct scatterlist outhdr, iv_sg, status_sg, **sgs;
int i;
u64 dst_len; u64 dst_len;
unsigned int num_out = 0, num_in = 0; unsigned int num_out = 0, num_in = 0;
int sg_total; int sg_total;
uint8_t *iv; uint8_t *iv;
struct scatterlist *sg;
src_nents = sg_nents_for_len(req->src, req->cryptlen); src_nents = sg_nents_for_len(req->src, req->cryptlen);
if (src_nents < 0) {
pr_err("Invalid number of src SG.\n");
return src_nents;
}
dst_nents = sg_nents(req->dst); dst_nents = sg_nents(req->dst);
pr_debug("virtio_crypto: Number of sgs (src_nents: %d, dst_nents: %d)\n", pr_debug("virtio_crypto: Number of sgs (src_nents: %d, dst_nents: %d)\n",
...@@ -402,6 +407,7 @@ __virtio_crypto_skcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req, ...@@ -402,6 +407,7 @@ __virtio_crypto_skcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req,
goto free; goto free;
} }
dst_len = min_t(unsigned int, req->cryptlen, dst_len);
pr_debug("virtio_crypto: src_len: %u, dst_len: %llu\n", pr_debug("virtio_crypto: src_len: %u, dst_len: %llu\n",
req->cryptlen, dst_len); req->cryptlen, dst_len);
...@@ -442,12 +448,12 @@ __virtio_crypto_skcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req, ...@@ -442,12 +448,12 @@ __virtio_crypto_skcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req,
vc_sym_req->iv = iv; vc_sym_req->iv = iv;
/* Source data */ /* Source data */
for (i = 0; i < src_nents; i++) for (sg = req->src; src_nents; sg = sg_next(sg), src_nents--)
sgs[num_out++] = &req->src[i]; sgs[num_out++] = sg;
/* Destination data */ /* Destination data */
for (i = 0; i < dst_nents; i++) for (sg = req->dst; sg; sg = sg_next(sg))
sgs[num_out + num_in++] = &req->dst[i]; sgs[num_out + num_in++] = sg;
/* Status */ /* Status */
sg_init_one(&status_sg, &vc_req->status, sizeof(vc_req->status)); sg_init_one(&status_sg, &vc_req->status, sizeof(vc_req->status));
...@@ -577,10 +583,11 @@ static void virtio_crypto_skcipher_finalize_req( ...@@ -577,10 +583,11 @@ static void virtio_crypto_skcipher_finalize_req(
scatterwalk_map_and_copy(req->iv, req->dst, scatterwalk_map_and_copy(req->iv, req->dst,
req->cryptlen - AES_BLOCK_SIZE, req->cryptlen - AES_BLOCK_SIZE,
AES_BLOCK_SIZE, 0); AES_BLOCK_SIZE, 0);
crypto_finalize_skcipher_request(vc_sym_req->base.dataq->engine,
req, err);
kzfree(vc_sym_req->iv); kzfree(vc_sym_req->iv);
virtcrypto_clear_request(&vc_sym_req->base); virtcrypto_clear_request(&vc_sym_req->base);
crypto_finalize_skcipher_request(vc_sym_req->base.dataq->engine,
req, err);
} }
static struct virtio_crypto_algo virtio_crypto_algs[] = { { static struct virtio_crypto_algo virtio_crypto_algs[] = { {
......
...@@ -116,7 +116,7 @@ config MIC_COSM ...@@ -116,7 +116,7 @@ config MIC_COSM
config VOP config VOP
tristate "VOP Driver" tristate "VOP Driver"
depends on VOP_BUS && VHOST_DPN depends on VOP_BUS
select VHOST_RING select VHOST_RING
select VIRTIO select VIRTIO
help help
......
...@@ -50,7 +50,7 @@ config CAIF_HSI ...@@ -50,7 +50,7 @@ config CAIF_HSI
config CAIF_VIRTIO config CAIF_VIRTIO
tristate "CAIF virtio transport driver" tristate "CAIF virtio transport driver"
depends on CAIF && HAS_DMA && VHOST_DPN depends on CAIF && HAS_DMA
select VHOST_RING select VHOST_RING
select VIRTIO select VIRTIO
select GENERIC_ALLOCATOR select GENERIC_ALLOCATOR
......
...@@ -10,7 +10,7 @@ if VDPA ...@@ -10,7 +10,7 @@ if VDPA
config VDPA_SIM config VDPA_SIM
tristate "vDPA device simulator" tristate "vDPA device simulator"
depends on RUNTIME_TESTING_MENU && HAS_DMA && VHOST_DPN depends on RUNTIME_TESTING_MENU && HAS_DMA
select VHOST_RING select VHOST_RING
default n default n
help help
......
...@@ -185,6 +185,9 @@ void ifcvf_set_status(struct ifcvf_hw *hw, u8 status) ...@@ -185,6 +185,9 @@ void ifcvf_set_status(struct ifcvf_hw *hw, u8 status)
void ifcvf_reset(struct ifcvf_hw *hw) void ifcvf_reset(struct ifcvf_hw *hw)
{ {
hw->config_cb.callback = NULL;
hw->config_cb.private = NULL;
ifcvf_set_status(hw, 0); ifcvf_set_status(hw, 0);
/* flush set_status, make sure VF is stopped, reset */ /* flush set_status, make sure VF is stopped, reset */
ifcvf_get_status(hw); ifcvf_get_status(hw);
......
...@@ -27,6 +27,7 @@ ...@@ -27,6 +27,7 @@
((1ULL << VIRTIO_NET_F_MAC) | \ ((1ULL << VIRTIO_NET_F_MAC) | \
(1ULL << VIRTIO_F_ANY_LAYOUT) | \ (1ULL << VIRTIO_F_ANY_LAYOUT) | \
(1ULL << VIRTIO_F_VERSION_1) | \ (1ULL << VIRTIO_F_VERSION_1) | \
(1ULL << VIRTIO_NET_F_STATUS) | \
(1ULL << VIRTIO_F_ORDER_PLATFORM) | \ (1ULL << VIRTIO_F_ORDER_PLATFORM) | \
(1ULL << VIRTIO_F_IOMMU_PLATFORM) | \ (1ULL << VIRTIO_F_IOMMU_PLATFORM) | \
(1ULL << VIRTIO_NET_F_MRG_RXBUF)) (1ULL << VIRTIO_NET_F_MRG_RXBUF))
...@@ -81,6 +82,9 @@ struct ifcvf_hw { ...@@ -81,6 +82,9 @@ struct ifcvf_hw {
void __iomem *net_cfg; void __iomem *net_cfg;
struct vring_info vring[IFCVF_MAX_QUEUE_PAIRS * 2]; struct vring_info vring[IFCVF_MAX_QUEUE_PAIRS * 2];
void __iomem * const *base; void __iomem * const *base;
char config_msix_name[256];
struct vdpa_callback config_cb;
}; };
struct ifcvf_adapter { struct ifcvf_adapter {
......
...@@ -18,6 +18,16 @@ ...@@ -18,6 +18,16 @@
#define DRIVER_AUTHOR "Intel Corporation" #define DRIVER_AUTHOR "Intel Corporation"
#define IFCVF_DRIVER_NAME "ifcvf" #define IFCVF_DRIVER_NAME "ifcvf"
static irqreturn_t ifcvf_config_changed(int irq, void *arg)
{
struct ifcvf_hw *vf = arg;
if (vf->config_cb.callback)
return vf->config_cb.callback(vf->config_cb.private);
return IRQ_HANDLED;
}
static irqreturn_t ifcvf_intr_handler(int irq, void *arg) static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
{ {
struct vring_info *vring = arg; struct vring_info *vring = arg;
...@@ -28,6 +38,68 @@ static irqreturn_t ifcvf_intr_handler(int irq, void *arg) ...@@ -28,6 +38,68 @@ static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
return IRQ_HANDLED; return IRQ_HANDLED;
} }
static void ifcvf_free_irq_vectors(void *data)
{
pci_free_irq_vectors(data);
}
static void ifcvf_free_irq(struct ifcvf_adapter *adapter, int queues)
{
struct pci_dev *pdev = adapter->pdev;
struct ifcvf_hw *vf = &adapter->vf;
int i;
for (i = 0; i < queues; i++)
devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]);
ifcvf_free_irq_vectors(pdev);
}
static int ifcvf_request_irq(struct ifcvf_adapter *adapter)
{
struct pci_dev *pdev = adapter->pdev;
struct ifcvf_hw *vf = &adapter->vf;
int vector, i, ret, irq;
ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR,
IFCVF_MAX_INTR, PCI_IRQ_MSIX);
if (ret < 0) {
IFCVF_ERR(pdev, "Failed to alloc IRQ vectors\n");
return ret;
}
snprintf(vf->config_msix_name, 256, "ifcvf[%s]-config\n",
pci_name(pdev));
vector = 0;
irq = pci_irq_vector(pdev, vector);
ret = devm_request_irq(&pdev->dev, irq,
ifcvf_config_changed, 0,
vf->config_msix_name, vf);
for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n",
pci_name(pdev), i);
vector = i + IFCVF_MSI_QUEUE_OFF;
irq = pci_irq_vector(pdev, vector);
ret = devm_request_irq(&pdev->dev, irq,
ifcvf_intr_handler, 0,
vf->vring[i].msix_name,
&vf->vring[i]);
if (ret) {
IFCVF_ERR(pdev,
"Failed to request irq for vq %d\n", i);
ifcvf_free_irq(adapter, i);
return ret;
}
vf->vring[i].irq = irq;
}
return 0;
}
static int ifcvf_start_datapath(void *private) static int ifcvf_start_datapath(void *private)
{ {
struct ifcvf_hw *vf = ifcvf_private_to_vf(private); struct ifcvf_hw *vf = ifcvf_private_to_vf(private);
...@@ -118,17 +190,37 @@ static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status) ...@@ -118,17 +190,37 @@ static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
{ {
struct ifcvf_adapter *adapter; struct ifcvf_adapter *adapter;
struct ifcvf_hw *vf; struct ifcvf_hw *vf;
u8 status_old;
int ret;
vf = vdpa_to_vf(vdpa_dev); vf = vdpa_to_vf(vdpa_dev);
adapter = dev_get_drvdata(vdpa_dev->dev.parent); adapter = dev_get_drvdata(vdpa_dev->dev.parent);
status_old = ifcvf_get_status(vf);
if (status == 0) { if (status_old == status)
return;
if ((status_old & VIRTIO_CONFIG_S_DRIVER_OK) &&
!(status & VIRTIO_CONFIG_S_DRIVER_OK)) {
ifcvf_stop_datapath(adapter); ifcvf_stop_datapath(adapter);
ifcvf_free_irq(adapter, IFCVF_MAX_QUEUE_PAIRS * 2);
}
if (status == 0) {
ifcvf_reset_vring(adapter); ifcvf_reset_vring(adapter);
return; return;
} }
if (status & VIRTIO_CONFIG_S_DRIVER_OK) { if ((status & VIRTIO_CONFIG_S_DRIVER_OK) &&
!(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) {
ret = ifcvf_request_irq(adapter);
if (ret) {
status = ifcvf_get_status(vf);
status |= VIRTIO_CONFIG_S_FAILED;
ifcvf_set_status(vf, status);
return;
}
if (ifcvf_start_datapath(adapter) < 0) if (ifcvf_start_datapath(adapter) < 0)
IFCVF_ERR(adapter->pdev, IFCVF_ERR(adapter->pdev,
"Failed to set ifcvf vdpa status %u\n", "Failed to set ifcvf vdpa status %u\n",
...@@ -254,7 +346,10 @@ static void ifcvf_vdpa_set_config(struct vdpa_device *vdpa_dev, ...@@ -254,7 +346,10 @@ static void ifcvf_vdpa_set_config(struct vdpa_device *vdpa_dev,
static void ifcvf_vdpa_set_config_cb(struct vdpa_device *vdpa_dev, static void ifcvf_vdpa_set_config_cb(struct vdpa_device *vdpa_dev,
struct vdpa_callback *cb) struct vdpa_callback *cb)
{ {
/* We don't support config interrupt */ struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
vf->config_cb.callback = cb->callback;
vf->config_cb.private = cb->private;
} }
/* /*
...@@ -284,38 +379,6 @@ static const struct vdpa_config_ops ifc_vdpa_ops = { ...@@ -284,38 +379,6 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {
.set_config_cb = ifcvf_vdpa_set_config_cb, .set_config_cb = ifcvf_vdpa_set_config_cb,
}; };
static int ifcvf_request_irq(struct ifcvf_adapter *adapter)
{
struct pci_dev *pdev = adapter->pdev;
struct ifcvf_hw *vf = &adapter->vf;
int vector, i, ret, irq;
for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n",
pci_name(pdev), i);
vector = i + IFCVF_MSI_QUEUE_OFF;
irq = pci_irq_vector(pdev, vector);
ret = devm_request_irq(&pdev->dev, irq,
ifcvf_intr_handler, 0,
vf->vring[i].msix_name,
&vf->vring[i]);
if (ret) {
IFCVF_ERR(pdev,
"Failed to request irq for vq %d\n", i);
return ret;
}
vf->vring[i].irq = irq;
}
return 0;
}
static void ifcvf_free_irq_vectors(void *data)
{
pci_free_irq_vectors(data);
}
static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id) static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{ {
struct device *dev = &pdev->dev; struct device *dev = &pdev->dev;
...@@ -349,13 +412,6 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id) ...@@ -349,13 +412,6 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
return ret; return ret;
} }
ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR,
IFCVF_MAX_INTR, PCI_IRQ_MSIX);
if (ret < 0) {
IFCVF_ERR(pdev, "Failed to alloc irq vectors\n");
return ret;
}
ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev); ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);
if (ret) { if (ret) {
IFCVF_ERR(pdev, IFCVF_ERR(pdev,
...@@ -379,12 +435,6 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id) ...@@ -379,12 +435,6 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
adapter->pdev = pdev; adapter->pdev = pdev;
adapter->vdpa.dma_dev = &pdev->dev; adapter->vdpa.dma_dev = &pdev->dev;
ret = ifcvf_request_irq(adapter);
if (ret) {
IFCVF_ERR(pdev, "Failed to request MSI-X irq\n");
goto err;
}
ret = ifcvf_init_hw(vf, pdev); ret = ifcvf_init_hw(vf, pdev);
if (ret) { if (ret) {
IFCVF_ERR(pdev, "Failed to init IFCVF hw\n"); IFCVF_ERR(pdev, "Failed to init IFCVF hw\n");
......
...@@ -101,7 +101,7 @@ static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx) ...@@ -101,7 +101,7 @@ static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
static void vdpasim_vq_reset(struct vdpasim_virtqueue *vq) static void vdpasim_vq_reset(struct vdpasim_virtqueue *vq)
{ {
vq->ready = 0; vq->ready = false;
vq->desc_addr = 0; vq->desc_addr = 0;
vq->driver_addr = 0; vq->driver_addr = 0;
vq->device_addr = 0; vq->device_addr = 0;
...@@ -131,9 +131,10 @@ static void vdpasim_work(struct work_struct *work) ...@@ -131,9 +131,10 @@ static void vdpasim_work(struct work_struct *work)
vdpasim, work); vdpasim, work);
struct vdpasim_virtqueue *txq = &vdpasim->vqs[1]; struct vdpasim_virtqueue *txq = &vdpasim->vqs[1];
struct vdpasim_virtqueue *rxq = &vdpasim->vqs[0]; struct vdpasim_virtqueue *rxq = &vdpasim->vqs[0];
size_t read, write, total_write; ssize_t read, write;
int err; size_t total_write;
int pkts = 0; int pkts = 0;
int err;
spin_lock(&vdpasim->lock); spin_lock(&vdpasim->lock);
......
...@@ -13,15 +13,6 @@ config VHOST_RING ...@@ -13,15 +13,6 @@ config VHOST_RING
This option is selected by any driver which needs to access This option is selected by any driver which needs to access
the host side of a virtio ring. the host side of a virtio ring.
config VHOST_DPN
bool
depends on !ARM || AEABI
default y
help
Anything selecting VHOST or VHOST_RING must depend on VHOST_DPN.
This excludes the deprecated ARM ABI since that forces a 4 byte
alignment on all structs - incompatible with virtio spec requirements.
config VHOST config VHOST
tristate tristate
select VHOST_IOTLB select VHOST_IOTLB
...@@ -37,7 +28,7 @@ if VHOST_MENU ...@@ -37,7 +28,7 @@ if VHOST_MENU
config VHOST_NET config VHOST_NET
tristate "Host kernel accelerator for virtio net" tristate "Host kernel accelerator for virtio net"
depends on NET && EVENTFD && (TUN || !TUN) && (TAP || !TAP) && VHOST_DPN depends on NET && EVENTFD && (TUN || !TUN) && (TAP || !TAP)
select VHOST select VHOST
---help--- ---help---
This kernel module can be loaded in host kernel to accelerate This kernel module can be loaded in host kernel to accelerate
...@@ -49,7 +40,7 @@ config VHOST_NET ...@@ -49,7 +40,7 @@ config VHOST_NET
config VHOST_SCSI config VHOST_SCSI
tristate "VHOST_SCSI TCM fabric driver" tristate "VHOST_SCSI TCM fabric driver"
depends on TARGET_CORE && EVENTFD && VHOST_DPN depends on TARGET_CORE && EVENTFD
select VHOST select VHOST
default n default n
---help--- ---help---
...@@ -58,7 +49,7 @@ config VHOST_SCSI ...@@ -58,7 +49,7 @@ config VHOST_SCSI
config VHOST_VSOCK config VHOST_VSOCK
tristate "vhost virtio-vsock driver" tristate "vhost virtio-vsock driver"
depends on VSOCKETS && EVENTFD && VHOST_DPN depends on VSOCKETS && EVENTFD
select VHOST select VHOST
select VIRTIO_VSOCKETS_COMMON select VIRTIO_VSOCKETS_COMMON
default n default n
...@@ -72,7 +63,7 @@ config VHOST_VSOCK ...@@ -72,7 +63,7 @@ config VHOST_VSOCK
config VHOST_VDPA config VHOST_VDPA
tristate "Vhost driver for vDPA-based backend" tristate "Vhost driver for vDPA-based backend"
depends on EVENTFD && VHOST_DPN depends on EVENTFD
select VHOST select VHOST
depends on VDPA depends on VDPA
help help
......
...@@ -1327,7 +1327,7 @@ static int vhost_net_open(struct inode *inode, struct file *f) ...@@ -1327,7 +1327,7 @@ static int vhost_net_open(struct inode *inode, struct file *f)
} }
vhost_dev_init(dev, vqs, VHOST_NET_VQ_MAX, vhost_dev_init(dev, vqs, VHOST_NET_VQ_MAX,
UIO_MAXIOV + VHOST_NET_BATCH, UIO_MAXIOV + VHOST_NET_BATCH,
VHOST_NET_PKT_WEIGHT, VHOST_NET_WEIGHT, VHOST_NET_PKT_WEIGHT, VHOST_NET_WEIGHT, true,
NULL); NULL);
vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, EPOLLOUT, dev); vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, EPOLLOUT, dev);
......
...@@ -1628,7 +1628,7 @@ static int vhost_scsi_open(struct inode *inode, struct file *f) ...@@ -1628,7 +1628,7 @@ static int vhost_scsi_open(struct inode *inode, struct file *f)
vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick; vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick;
} }
vhost_dev_init(&vs->dev, vqs, VHOST_SCSI_MAX_VQ, UIO_MAXIOV, vhost_dev_init(&vs->dev, vqs, VHOST_SCSI_MAX_VQ, UIO_MAXIOV,
VHOST_SCSI_WEIGHT, 0, NULL); VHOST_SCSI_WEIGHT, 0, true, NULL);
vhost_scsi_init_inflight(vs, NULL); vhost_scsi_init_inflight(vs, NULL);
......
...@@ -120,7 +120,7 @@ static int vhost_test_open(struct inode *inode, struct file *f) ...@@ -120,7 +120,7 @@ static int vhost_test_open(struct inode *inode, struct file *f)
vqs[VHOST_TEST_VQ] = &n->vqs[VHOST_TEST_VQ]; vqs[VHOST_TEST_VQ] = &n->vqs[VHOST_TEST_VQ];
n->vqs[VHOST_TEST_VQ].handle_kick = handle_vq_kick; n->vqs[VHOST_TEST_VQ].handle_kick = handle_vq_kick;
vhost_dev_init(dev, vqs, VHOST_TEST_VQ_MAX, UIO_MAXIOV, vhost_dev_init(dev, vqs, VHOST_TEST_VQ_MAX, UIO_MAXIOV,
VHOST_TEST_PKT_WEIGHT, VHOST_TEST_WEIGHT, NULL); VHOST_TEST_PKT_WEIGHT, VHOST_TEST_WEIGHT, true, NULL);
f->private_data = n; f->private_data = n;
......
...@@ -15,12 +15,14 @@ ...@@ -15,12 +15,14 @@
#include <linux/module.h> #include <linux/module.h>
#include <linux/cdev.h> #include <linux/cdev.h>
#include <linux/device.h> #include <linux/device.h>
#include <linux/mm.h>
#include <linux/iommu.h> #include <linux/iommu.h>
#include <linux/uuid.h> #include <linux/uuid.h>
#include <linux/vdpa.h> #include <linux/vdpa.h>
#include <linux/nospec.h> #include <linux/nospec.h>
#include <linux/vhost.h> #include <linux/vhost.h>
#include <linux/virtio_net.h> #include <linux/virtio_net.h>
#include <linux/kernel.h>
#include "vhost.h" #include "vhost.h"
...@@ -70,6 +72,7 @@ struct vhost_vdpa { ...@@ -70,6 +72,7 @@ struct vhost_vdpa {
int nvqs; int nvqs;
int virtio_id; int virtio_id;
int minor; int minor;
struct eventfd_ctx *config_ctx;
}; };
static DEFINE_IDA(vhost_vdpa_ida); static DEFINE_IDA(vhost_vdpa_ida);
...@@ -101,6 +104,17 @@ static irqreturn_t vhost_vdpa_virtqueue_cb(void *private) ...@@ -101,6 +104,17 @@ static irqreturn_t vhost_vdpa_virtqueue_cb(void *private)
return IRQ_HANDLED; return IRQ_HANDLED;
} }
static irqreturn_t vhost_vdpa_config_cb(void *private)
{
struct vhost_vdpa *v = private;
struct eventfd_ctx *config_ctx = v->config_ctx;
if (config_ctx)
eventfd_signal(config_ctx, 1);
return IRQ_HANDLED;
}
static void vhost_vdpa_reset(struct vhost_vdpa *v) static void vhost_vdpa_reset(struct vhost_vdpa *v)
{ {
struct vdpa_device *vdpa = v->vdpa; struct vdpa_device *vdpa = v->vdpa;
...@@ -288,6 +302,36 @@ static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp) ...@@ -288,6 +302,36 @@ static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp)
return 0; return 0;
} }
static void vhost_vdpa_config_put(struct vhost_vdpa *v)
{
if (v->config_ctx)
eventfd_ctx_put(v->config_ctx);
}
static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp)
{
struct vdpa_callback cb;
int fd;
struct eventfd_ctx *ctx;
cb.callback = vhost_vdpa_config_cb;
cb.private = v->vdpa;
if (copy_from_user(&fd, argp, sizeof(fd)))
return -EFAULT;
ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
swap(ctx, v->config_ctx);
if (!IS_ERR_OR_NULL(ctx))
eventfd_ctx_put(ctx);
if (IS_ERR(v->config_ctx))
return PTR_ERR(v->config_ctx);
v->vdpa->config->set_config_cb(v->vdpa, &cb);
return 0;
}
static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
void __user *argp) void __user *argp)
{ {
...@@ -395,6 +439,9 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, ...@@ -395,6 +439,9 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep,
case VHOST_SET_LOG_FD: case VHOST_SET_LOG_FD:
r = -ENOIOCTLCMD; r = -ENOIOCTLCMD;
break; break;
case VHOST_VDPA_SET_CONFIG_CALL:
r = vhost_vdpa_set_config_call(v, argp);
break;
default: default:
r = vhost_dev_ioctl(&v->vdev, cmd, argp); r = vhost_dev_ioctl(&v->vdev, cmd, argp);
if (r == -ENOIOCTLCMD) if (r == -ENOIOCTLCMD)
...@@ -694,7 +741,7 @@ static int vhost_vdpa_open(struct inode *inode, struct file *filep) ...@@ -694,7 +741,7 @@ static int vhost_vdpa_open(struct inode *inode, struct file *filep)
vqs[i] = &v->vqs[i]; vqs[i] = &v->vqs[i];
vqs[i]->handle_kick = handle_vq_kick; vqs[i]->handle_kick = handle_vq_kick;
} }
vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false,
vhost_vdpa_process_iotlb_msg); vhost_vdpa_process_iotlb_msg);
dev->iotlb = vhost_iotlb_alloc(0, 0); dev->iotlb = vhost_iotlb_alloc(0, 0);
...@@ -729,6 +776,7 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep) ...@@ -729,6 +776,7 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep)
vhost_dev_stop(&v->vdev); vhost_dev_stop(&v->vdev);
vhost_vdpa_iotlb_free(v); vhost_vdpa_iotlb_free(v);
vhost_vdpa_free_domain(v); vhost_vdpa_free_domain(v);
vhost_vdpa_config_put(v);
vhost_dev_cleanup(&v->vdev); vhost_dev_cleanup(&v->vdev);
kfree(v->vdev.vqs); kfree(v->vdev.vqs);
mutex_unlock(&d->mutex); mutex_unlock(&d->mutex);
...@@ -739,12 +787,74 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep) ...@@ -739,12 +787,74 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep)
return 0; return 0;
} }
#ifdef CONFIG_MMU
static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf)
{
struct vhost_vdpa *v = vmf->vma->vm_file->private_data;
struct vdpa_device *vdpa = v->vdpa;
const struct vdpa_config_ops *ops = vdpa->config;
struct vdpa_notification_area notify;
struct vm_area_struct *vma = vmf->vma;
u16 index = vma->vm_pgoff;
notify = ops->get_vq_notification(vdpa, index);
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
if (remap_pfn_range(vma, vmf->address & PAGE_MASK,
notify.addr >> PAGE_SHIFT, PAGE_SIZE,
vma->vm_page_prot))
return VM_FAULT_SIGBUS;
return VM_FAULT_NOPAGE;
}
static const struct vm_operations_struct vhost_vdpa_vm_ops = {
.fault = vhost_vdpa_fault,
};
static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma)
{
struct vhost_vdpa *v = vma->vm_file->private_data;
struct vdpa_device *vdpa = v->vdpa;
const struct vdpa_config_ops *ops = vdpa->config;
struct vdpa_notification_area notify;
int index = vma->vm_pgoff;
if (vma->vm_end - vma->vm_start != PAGE_SIZE)
return -EINVAL;
if ((vma->vm_flags & VM_SHARED) == 0)
return -EINVAL;
if (vma->vm_flags & VM_READ)
return -EINVAL;
if (index > 65535)
return -EINVAL;
if (!ops->get_vq_notification)
return -ENOTSUPP;
/* To be safe and easily modelled by userspace, We only
* support the doorbell which sits on the page boundary and
* does not share the page with other registers.
*/
notify = ops->get_vq_notification(vdpa, index);
if (notify.addr & (PAGE_SIZE - 1))
return -EINVAL;
if (vma->vm_end - vma->vm_start != notify.size)
return -ENOTSUPP;
vma->vm_ops = &vhost_vdpa_vm_ops;
return 0;
}
#endif /* CONFIG_MMU */
static const struct file_operations vhost_vdpa_fops = { static const struct file_operations vhost_vdpa_fops = {
.owner = THIS_MODULE, .owner = THIS_MODULE,
.open = vhost_vdpa_open, .open = vhost_vdpa_open,
.release = vhost_vdpa_release, .release = vhost_vdpa_release,
.write_iter = vhost_vdpa_chr_write_iter, .write_iter = vhost_vdpa_chr_write_iter,
.unlocked_ioctl = vhost_vdpa_unlocked_ioctl, .unlocked_ioctl = vhost_vdpa_unlocked_ioctl,
#ifdef CONFIG_MMU
.mmap = vhost_vdpa_mmap,
#endif /* CONFIG_MMU */
.compat_ioctl = compat_ptr_ioctl, .compat_ioctl = compat_ptr_ioctl,
}; };
......
...@@ -166,11 +166,16 @@ static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, ...@@ -166,11 +166,16 @@ static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync,
void *key) void *key)
{ {
struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait); struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait);
struct vhost_work *work = &poll->work;
if (!(key_to_poll(key) & poll->mask)) if (!(key_to_poll(key) & poll->mask))
return 0; return 0;
vhost_poll_queue(poll); if (!poll->dev->use_worker)
work->fn(work);
else
vhost_poll_queue(poll);
return 0; return 0;
} }
...@@ -454,6 +459,7 @@ static size_t vhost_get_desc_size(struct vhost_virtqueue *vq, ...@@ -454,6 +459,7 @@ static size_t vhost_get_desc_size(struct vhost_virtqueue *vq,
void vhost_dev_init(struct vhost_dev *dev, void vhost_dev_init(struct vhost_dev *dev,
struct vhost_virtqueue **vqs, int nvqs, struct vhost_virtqueue **vqs, int nvqs,
int iov_limit, int weight, int byte_weight, int iov_limit, int weight, int byte_weight,
bool use_worker,
int (*msg_handler)(struct vhost_dev *dev, int (*msg_handler)(struct vhost_dev *dev,
struct vhost_iotlb_msg *msg)) struct vhost_iotlb_msg *msg))
{ {
...@@ -471,6 +477,7 @@ void vhost_dev_init(struct vhost_dev *dev, ...@@ -471,6 +477,7 @@ void vhost_dev_init(struct vhost_dev *dev,
dev->iov_limit = iov_limit; dev->iov_limit = iov_limit;
dev->weight = weight; dev->weight = weight;
dev->byte_weight = byte_weight; dev->byte_weight = byte_weight;
dev->use_worker = use_worker;
dev->msg_handler = msg_handler; dev->msg_handler = msg_handler;
init_llist_head(&dev->work_list); init_llist_head(&dev->work_list);
init_waitqueue_head(&dev->wait); init_waitqueue_head(&dev->wait);
...@@ -534,6 +541,36 @@ bool vhost_dev_has_owner(struct vhost_dev *dev) ...@@ -534,6 +541,36 @@ bool vhost_dev_has_owner(struct vhost_dev *dev)
} }
EXPORT_SYMBOL_GPL(vhost_dev_has_owner); EXPORT_SYMBOL_GPL(vhost_dev_has_owner);
static void vhost_attach_mm(struct vhost_dev *dev)
{
/* No owner, become one */
if (dev->use_worker) {
dev->mm = get_task_mm(current);
} else {
/* vDPA device does not use worker thead, so there's
* no need to hold the address space for mm. This help
* to avoid deadlock in the case of mmap() which may
* held the refcnt of the file and depends on release
* method to remove vma.
*/
dev->mm = current->mm;
mmgrab(dev->mm);
}
}
static void vhost_detach_mm(struct vhost_dev *dev)
{
if (!dev->mm)
return;
if (dev->use_worker)
mmput(dev->mm);
else
mmdrop(dev->mm);
dev->mm = NULL;
}
/* Caller should have device mutex */ /* Caller should have device mutex */
long vhost_dev_set_owner(struct vhost_dev *dev) long vhost_dev_set_owner(struct vhost_dev *dev)
{ {
...@@ -546,21 +583,24 @@ long vhost_dev_set_owner(struct vhost_dev *dev) ...@@ -546,21 +583,24 @@ long vhost_dev_set_owner(struct vhost_dev *dev)
goto err_mm; goto err_mm;
} }
/* No owner, become one */ vhost_attach_mm(dev);
dev->mm = get_task_mm(current);
dev->kcov_handle = kcov_common_handle(); dev->kcov_handle = kcov_common_handle();
worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid); if (dev->use_worker) {
if (IS_ERR(worker)) { worker = kthread_create(vhost_worker, dev,
err = PTR_ERR(worker); "vhost-%d", current->pid);
goto err_worker; if (IS_ERR(worker)) {
} err = PTR_ERR(worker);
goto err_worker;
}
dev->worker = worker; dev->worker = worker;
wake_up_process(worker); /* avoid contributing to loadavg */ wake_up_process(worker); /* avoid contributing to loadavg */
err = vhost_attach_cgroups(dev); err = vhost_attach_cgroups(dev);
if (err) if (err)
goto err_cgroup; goto err_cgroup;
}
err = vhost_dev_alloc_iovecs(dev); err = vhost_dev_alloc_iovecs(dev);
if (err) if (err)
...@@ -568,12 +608,12 @@ long vhost_dev_set_owner(struct vhost_dev *dev) ...@@ -568,12 +608,12 @@ long vhost_dev_set_owner(struct vhost_dev *dev)
return 0; return 0;
err_cgroup: err_cgroup:
kthread_stop(worker); if (dev->worker) {
dev->worker = NULL; kthread_stop(dev->worker);
dev->worker = NULL;
}
err_worker: err_worker:
if (dev->mm) vhost_detach_mm(dev);
mmput(dev->mm);
dev->mm = NULL;
dev->kcov_handle = 0; dev->kcov_handle = 0;
err_mm: err_mm:
return err; return err;
...@@ -670,9 +710,7 @@ void vhost_dev_cleanup(struct vhost_dev *dev) ...@@ -670,9 +710,7 @@ void vhost_dev_cleanup(struct vhost_dev *dev)
dev->worker = NULL; dev->worker = NULL;
dev->kcov_handle = 0; dev->kcov_handle = 0;
} }
if (dev->mm) vhost_detach_mm(dev);
mmput(dev->mm);
dev->mm = NULL;
} }
EXPORT_SYMBOL_GPL(vhost_dev_cleanup); EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
...@@ -882,7 +920,7 @@ static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq, ...@@ -882,7 +920,7 @@ static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq,
#define vhost_put_user(vq, x, ptr) \ #define vhost_put_user(vq, x, ptr) \
({ \ ({ \
int ret = -EFAULT; \ int ret; \
if (!vq->iotlb) { \ if (!vq->iotlb) { \
ret = __put_user(x, ptr); \ ret = __put_user(x, ptr); \
} else { \ } else { \
...@@ -1244,9 +1282,9 @@ static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access) ...@@ -1244,9 +1282,9 @@ static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access)
} }
static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num, static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num,
struct vring_desc __user *desc, vring_desc_t __user *desc,
struct vring_avail __user *avail, vring_avail_t __user *avail,
struct vring_used __user *used) vring_used_t __user *used)
{ {
return access_ok(desc, vhost_get_desc_size(vq, num)) && return access_ok(desc, vhost_get_desc_size(vq, num)) &&
...@@ -1574,7 +1612,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg ...@@ -1574,7 +1612,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg
r = -EFAULT; r = -EFAULT;
break; break;
} }
eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd); eventfp = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_fget(f.fd);
if (IS_ERR(eventfp)) { if (IS_ERR(eventfp)) {
r = PTR_ERR(eventfp); r = PTR_ERR(eventfp);
break; break;
...@@ -1590,7 +1628,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg ...@@ -1590,7 +1628,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg
r = -EFAULT; r = -EFAULT;
break; break;
} }
ctx = f.fd == -1 ? NULL : eventfd_ctx_fdget(f.fd); ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd);
if (IS_ERR(ctx)) { if (IS_ERR(ctx)) {
r = PTR_ERR(ctx); r = PTR_ERR(ctx);
break; break;
...@@ -1602,7 +1640,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg ...@@ -1602,7 +1640,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg
r = -EFAULT; r = -EFAULT;
break; break;
} }
ctx = f.fd == -1 ? NULL : eventfd_ctx_fdget(f.fd); ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd);
if (IS_ERR(ctx)) { if (IS_ERR(ctx)) {
r = PTR_ERR(ctx); r = PTR_ERR(ctx);
break; break;
...@@ -1727,7 +1765,7 @@ long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp) ...@@ -1727,7 +1765,7 @@ long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
r = get_user(fd, (int __user *)argp); r = get_user(fd, (int __user *)argp);
if (r < 0) if (r < 0)
break; break;
ctx = fd == -1 ? NULL : eventfd_ctx_fdget(fd); ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
if (IS_ERR(ctx)) { if (IS_ERR(ctx)) {
r = PTR_ERR(ctx); r = PTR_ERR(ctx);
break; break;
...@@ -2300,7 +2338,7 @@ static int __vhost_add_used_n(struct vhost_virtqueue *vq, ...@@ -2300,7 +2338,7 @@ static int __vhost_add_used_n(struct vhost_virtqueue *vq,
struct vring_used_elem *heads, struct vring_used_elem *heads,
unsigned count) unsigned count)
{ {
struct vring_used_elem __user *used; vring_used_elem_t __user *used;
u16 old, new; u16 old, new;
int start; int start;
......
...@@ -67,9 +67,9 @@ struct vhost_virtqueue { ...@@ -67,9 +67,9 @@ struct vhost_virtqueue {
/* The actual ring of buffers. */ /* The actual ring of buffers. */
struct mutex mutex; struct mutex mutex;
unsigned int num; unsigned int num;
struct vring_desc __user *desc; vring_desc_t __user *desc;
struct vring_avail __user *avail; vring_avail_t __user *avail;
struct vring_used __user *used; vring_used_t __user *used;
const struct vhost_iotlb_map *meta_iotlb[VHOST_NUM_ADDRS]; const struct vhost_iotlb_map *meta_iotlb[VHOST_NUM_ADDRS];
struct file *kick; struct file *kick;
struct eventfd_ctx *call_ctx; struct eventfd_ctx *call_ctx;
...@@ -154,6 +154,7 @@ struct vhost_dev { ...@@ -154,6 +154,7 @@ struct vhost_dev {
int weight; int weight;
int byte_weight; int byte_weight;
u64 kcov_handle; u64 kcov_handle;
bool use_worker;
int (*msg_handler)(struct vhost_dev *dev, int (*msg_handler)(struct vhost_dev *dev,
struct vhost_iotlb_msg *msg); struct vhost_iotlb_msg *msg);
}; };
...@@ -161,6 +162,7 @@ struct vhost_dev { ...@@ -161,6 +162,7 @@ struct vhost_dev {
bool vhost_exceeds_weight(struct vhost_virtqueue *vq, int pkts, int total_len); bool vhost_exceeds_weight(struct vhost_virtqueue *vq, int pkts, int total_len);
void vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue **vqs, void vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue **vqs,
int nvqs, int iov_limit, int weight, int byte_weight, int nvqs, int iov_limit, int weight, int byte_weight,
bool use_worker,
int (*msg_handler)(struct vhost_dev *dev, int (*msg_handler)(struct vhost_dev *dev,
struct vhost_iotlb_msg *msg)); struct vhost_iotlb_msg *msg));
long vhost_dev_set_owner(struct vhost_dev *dev); long vhost_dev_set_owner(struct vhost_dev *dev);
......
...@@ -620,9 +620,9 @@ static inline int xfer_to_user(const struct vringh *vrh, ...@@ -620,9 +620,9 @@ static inline int xfer_to_user(const struct vringh *vrh,
*/ */
int vringh_init_user(struct vringh *vrh, u64 features, int vringh_init_user(struct vringh *vrh, u64 features,
unsigned int num, bool weak_barriers, unsigned int num, bool weak_barriers,
struct vring_desc __user *desc, vring_desc_t __user *desc,
struct vring_avail __user *avail, vring_avail_t __user *avail,
struct vring_used __user *used) vring_used_t __user *used)
{ {
/* Sane power of 2 please! */ /* Sane power of 2 please! */
if (!num || num > 0xffff || (num & (num - 1))) { if (!num || num > 0xffff || (num & (num - 1))) {
......
...@@ -632,7 +632,7 @@ static int vhost_vsock_dev_open(struct inode *inode, struct file *file) ...@@ -632,7 +632,7 @@ static int vhost_vsock_dev_open(struct inode *inode, struct file *file)
vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs), vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs),
UIO_MAXIOV, VHOST_VSOCK_PKT_WEIGHT, UIO_MAXIOV, VHOST_VSOCK_PKT_WEIGHT,
VHOST_VSOCK_WEIGHT, NULL); VHOST_VSOCK_WEIGHT, true, NULL);
file->private_data = vsock; file->private_data = vsock;
spin_lock_init(&vsock->send_pkt_list_lock); spin_lock_init(&vsock->send_pkt_list_lock);
......
...@@ -78,6 +78,23 @@ config VIRTIO_BALLOON ...@@ -78,6 +78,23 @@ config VIRTIO_BALLOON
If unsure, say M. If unsure, say M.
config VIRTIO_MEM
tristate "Virtio mem driver"
default m
depends on X86_64
depends on VIRTIO
depends on MEMORY_HOTPLUG_SPARSE
depends on MEMORY_HOTREMOVE
select CONTIG_ALLOC
help
This driver provides access to virtio-mem paravirtualized memory
devices, allowing to hotplug and hotunplug memory.
This driver was only tested under x86-64, but should theoretically
work on all architectures that support memory hotplug and hotremove.
If unsure, say M.
config VIRTIO_INPUT config VIRTIO_INPUT
tristate "Virtio input driver" tristate "Virtio input driver"
depends on VIRTIO depends on VIRTIO
......
...@@ -7,3 +7,4 @@ virtio_pci-$(CONFIG_VIRTIO_PCI_LEGACY) += virtio_pci_legacy.o ...@@ -7,3 +7,4 @@ virtio_pci-$(CONFIG_VIRTIO_PCI_LEGACY) += virtio_pci_legacy.o
obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o
obj-$(CONFIG_VIRTIO_INPUT) += virtio_input.o obj-$(CONFIG_VIRTIO_INPUT) += virtio_input.o
obj-$(CONFIG_VIRTIO_VDPA) += virtio_vdpa.o obj-$(CONFIG_VIRTIO_VDPA) += virtio_vdpa.o
obj-$(CONFIG_VIRTIO_MEM) += virtio_mem.o
...@@ -1107,11 +1107,18 @@ static int virtballoon_restore(struct virtio_device *vdev) ...@@ -1107,11 +1107,18 @@ static int virtballoon_restore(struct virtio_device *vdev)
static int virtballoon_validate(struct virtio_device *vdev) static int virtballoon_validate(struct virtio_device *vdev)
{ {
/* Tell the host whether we care about poisoned pages. */ /*
* Inform the hypervisor that our pages are poisoned or
* initialized. If we cannot do that then we should disable
* page reporting as it could potentially change the contents
* of our free pages.
*/
if (!want_init_on_free() && if (!want_init_on_free() &&
(IS_ENABLED(CONFIG_PAGE_POISONING_NO_SANITY) || (IS_ENABLED(CONFIG_PAGE_POISONING_NO_SANITY) ||
!page_poisoning_enabled())) !page_poisoning_enabled()))
__virtio_clear_bit(vdev, VIRTIO_BALLOON_F_PAGE_POISON); __virtio_clear_bit(vdev, VIRTIO_BALLOON_F_PAGE_POISON);
else if (!virtio_has_feature(vdev, VIRTIO_BALLOON_F_PAGE_POISON))
__virtio_clear_bit(vdev, VIRTIO_BALLOON_F_REPORTING);
__virtio_clear_bit(vdev, VIRTIO_F_IOMMU_PLATFORM); __virtio_clear_bit(vdev, VIRTIO_F_IOMMU_PLATFORM);
return 0; return 0;
......
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Virtio-mem device driver.
*
* Copyright Red Hat, Inc. 2020
*
* Author(s): David Hildenbrand <david@redhat.com>
*/
#include <linux/virtio.h>
#include <linux/virtio_mem.h>
#include <linux/workqueue.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/memory_hotplug.h>
#include <linux/memory.h>
#include <linux/hrtimer.h>
#include <linux/crash_dump.h>
#include <linux/mutex.h>
#include <linux/bitmap.h>
#include <linux/lockdep.h>
#include <acpi/acpi_numa.h>
static bool unplug_online = true;
module_param(unplug_online, bool, 0644);
MODULE_PARM_DESC(unplug_online, "Try to unplug online memory");
enum virtio_mem_mb_state {
/* Unplugged, not added to Linux. Can be reused later. */
VIRTIO_MEM_MB_STATE_UNUSED = 0,
/* (Partially) plugged, not added to Linux. Error on add_memory(). */
VIRTIO_MEM_MB_STATE_PLUGGED,
/* Fully plugged, fully added to Linux, offline. */
VIRTIO_MEM_MB_STATE_OFFLINE,
/* Partially plugged, fully added to Linux, offline. */
VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL,
/* Fully plugged, fully added to Linux, online (!ZONE_MOVABLE). */
VIRTIO_MEM_MB_STATE_ONLINE,
/* Partially plugged, fully added to Linux, online (!ZONE_MOVABLE). */
VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL,
/*
* Fully plugged, fully added to Linux, online (ZONE_MOVABLE).
* We are not allowed to allocate (unplug) parts of this block that
* are not movable (similar to gigantic pages). We will never allow
* to online OFFLINE_PARTIAL to ZONE_MOVABLE (as they would contain
* unmovable parts).
*/
VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE,
VIRTIO_MEM_MB_STATE_COUNT
};
struct virtio_mem {
struct virtio_device *vdev;
/* We might first have to unplug all memory when starting up. */
bool unplug_all_required;
/* Workqueue that processes the plug/unplug requests. */
struct work_struct wq;
atomic_t config_changed;
/* Virtqueue for guest->host requests. */
struct virtqueue *vq;
/* Wait for a host response to a guest request. */
wait_queue_head_t host_resp;
/* Space for one guest request and the host response. */
struct virtio_mem_req req;
struct virtio_mem_resp resp;
/* The current size of the device. */
uint64_t plugged_size;
/* The requested size of the device. */
uint64_t requested_size;
/* The device block size (for communicating with the device). */
uint64_t device_block_size;
/* The translated node id. NUMA_NO_NODE in case not specified. */
int nid;
/* Physical start address of the memory region. */
uint64_t addr;
/* Maximum region size in bytes. */
uint64_t region_size;
/* The subblock size. */
uint64_t subblock_size;
/* The number of subblocks per memory block. */
uint32_t nb_sb_per_mb;
/* Id of the first memory block of this device. */
unsigned long first_mb_id;
/* Id of the last memory block of this device. */
unsigned long last_mb_id;
/* Id of the last usable memory block of this device. */
unsigned long last_usable_mb_id;
/* Id of the next memory bock to prepare when needed. */
unsigned long next_mb_id;
/* The parent resource for all memory added via this device. */
struct resource *parent_resource;
/* Summary of all memory block states. */
unsigned long nb_mb_state[VIRTIO_MEM_MB_STATE_COUNT];
#define VIRTIO_MEM_NB_OFFLINE_THRESHOLD 10
/*
* One byte state per memory block.
*
* Allocated via vmalloc(). When preparing new blocks, resized
* (alloc+copy+free) when needed (crossing pages with the next mb).
* (when crossing pages).
*
* With 128MB memory blocks, we have states for 512GB of memory in one
* page.
*/
uint8_t *mb_state;
/*
* $nb_sb_per_mb bit per memory block. Handled similar to mb_state.
*
* With 4MB subblocks, we manage 128GB of memory in one page.
*/
unsigned long *sb_bitmap;
/*
* Mutex that protects the nb_mb_state, mb_state, and sb_bitmap.
*
* When this lock is held the pointers can't change, ONLINE and
* OFFLINE blocks can't change the state and no subblocks will get
* plugged/unplugged.
*/
struct mutex hotplug_mutex;
bool hotplug_active;
/* An error occurred we cannot handle - stop processing requests. */
bool broken;
/* The driver is being removed. */
spinlock_t removal_lock;
bool removing;
/* Timer for retrying to plug/unplug memory. */
struct hrtimer retry_timer;
unsigned int retry_timer_ms;
#define VIRTIO_MEM_RETRY_TIMER_MIN_MS 50000
#define VIRTIO_MEM_RETRY_TIMER_MAX_MS 300000
/* Memory notifier (online/offline events). */
struct notifier_block memory_notifier;
/* Next device in the list of virtio-mem devices. */
struct list_head next;
};
/*
* We have to share a single online_page callback among all virtio-mem
* devices. We use RCU to iterate the list in the callback.
*/
static DEFINE_MUTEX(virtio_mem_mutex);
static LIST_HEAD(virtio_mem_devices);
static void virtio_mem_online_page_cb(struct page *page, unsigned int order);
/*
* Register a virtio-mem device so it will be considered for the online_page
* callback.
*/
static int register_virtio_mem_device(struct virtio_mem *vm)
{
int rc = 0;
/* First device registers the callback. */
mutex_lock(&virtio_mem_mutex);
if (list_empty(&virtio_mem_devices))
rc = set_online_page_callback(&virtio_mem_online_page_cb);
if (!rc)
list_add_rcu(&vm->next, &virtio_mem_devices);
mutex_unlock(&virtio_mem_mutex);
return rc;
}
/*
* Unregister a virtio-mem device so it will no longer be considered for the
* online_page callback.
*/
static void unregister_virtio_mem_device(struct virtio_mem *vm)
{
/* Last device unregisters the callback. */
mutex_lock(&virtio_mem_mutex);
list_del_rcu(&vm->next);
if (list_empty(&virtio_mem_devices))
restore_online_page_callback(&virtio_mem_online_page_cb);
mutex_unlock(&virtio_mem_mutex);
synchronize_rcu();
}
/*
* Calculate the memory block id of a given address.
*/
static unsigned long virtio_mem_phys_to_mb_id(unsigned long addr)
{
return addr / memory_block_size_bytes();
}
/*
* Calculate the physical start address of a given memory block id.
*/
static unsigned long virtio_mem_mb_id_to_phys(unsigned long mb_id)
{
return mb_id * memory_block_size_bytes();
}
/*
* Calculate the subblock id of a given address.
*/
static unsigned long virtio_mem_phys_to_sb_id(struct virtio_mem *vm,
unsigned long addr)
{
const unsigned long mb_id = virtio_mem_phys_to_mb_id(addr);
const unsigned long mb_addr = virtio_mem_mb_id_to_phys(mb_id);
return (addr - mb_addr) / vm->subblock_size;
}
/*
* Set the state of a memory block, taking care of the state counter.
*/
static void virtio_mem_mb_set_state(struct virtio_mem *vm, unsigned long mb_id,
enum virtio_mem_mb_state state)
{
const unsigned long idx = mb_id - vm->first_mb_id;
enum virtio_mem_mb_state old_state;
old_state = vm->mb_state[idx];
vm->mb_state[idx] = state;
BUG_ON(vm->nb_mb_state[old_state] == 0);
vm->nb_mb_state[old_state]--;
vm->nb_mb_state[state]++;
}
/*
* Get the state of a memory block.
*/
static enum virtio_mem_mb_state virtio_mem_mb_get_state(struct virtio_mem *vm,
unsigned long mb_id)
{
const unsigned long idx = mb_id - vm->first_mb_id;
return vm->mb_state[idx];
}
/*
* Prepare the state array for the next memory block.
*/
static int virtio_mem_mb_state_prepare_next_mb(struct virtio_mem *vm)
{
unsigned long old_bytes = vm->next_mb_id - vm->first_mb_id + 1;
unsigned long new_bytes = vm->next_mb_id - vm->first_mb_id + 2;
int old_pages = PFN_UP(old_bytes);
int new_pages = PFN_UP(new_bytes);
uint8_t *new_mb_state;
if (vm->mb_state && old_pages == new_pages)
return 0;
new_mb_state = vzalloc(new_pages * PAGE_SIZE);
if (!new_mb_state)
return -ENOMEM;
mutex_lock(&vm->hotplug_mutex);
if (vm->mb_state)
memcpy(new_mb_state, vm->mb_state, old_pages * PAGE_SIZE);
vfree(vm->mb_state);
vm->mb_state = new_mb_state;
mutex_unlock(&vm->hotplug_mutex);
return 0;
}
#define virtio_mem_for_each_mb_state(_vm, _mb_id, _state) \
for (_mb_id = _vm->first_mb_id; \
_mb_id < _vm->next_mb_id && _vm->nb_mb_state[_state]; \
_mb_id++) \
if (virtio_mem_mb_get_state(_vm, _mb_id) == _state)
#define virtio_mem_for_each_mb_state_rev(_vm, _mb_id, _state) \
for (_mb_id = _vm->next_mb_id - 1; \
_mb_id >= _vm->first_mb_id && _vm->nb_mb_state[_state]; \
_mb_id--) \
if (virtio_mem_mb_get_state(_vm, _mb_id) == _state)
/*
* Mark all selected subblocks plugged.
*
* Will not modify the state of the memory block.
*/
static void virtio_mem_mb_set_sb_plugged(struct virtio_mem *vm,
unsigned long mb_id, int sb_id,
int count)
{
const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
__bitmap_set(vm->sb_bitmap, bit, count);
}
/*
* Mark all selected subblocks unplugged.
*
* Will not modify the state of the memory block.
*/
static void virtio_mem_mb_set_sb_unplugged(struct virtio_mem *vm,
unsigned long mb_id, int sb_id,
int count)
{
const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
__bitmap_clear(vm->sb_bitmap, bit, count);
}
/*
* Test if all selected subblocks are plugged.
*/
static bool virtio_mem_mb_test_sb_plugged(struct virtio_mem *vm,
unsigned long mb_id, int sb_id,
int count)
{
const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
if (count == 1)
return test_bit(bit, vm->sb_bitmap);
/* TODO: Helper similar to bitmap_set() */
return find_next_zero_bit(vm->sb_bitmap, bit + count, bit) >=
bit + count;
}
/*
* Test if all selected subblocks are unplugged.
*/
static bool virtio_mem_mb_test_sb_unplugged(struct virtio_mem *vm,
unsigned long mb_id, int sb_id,
int count)
{
const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
/* TODO: Helper similar to bitmap_set() */
return find_next_bit(vm->sb_bitmap, bit + count, bit) >= bit + count;
}
/*
* Find the first unplugged subblock. Returns vm->nb_sb_per_mb in case there is
* none.
*/
static int virtio_mem_mb_first_unplugged_sb(struct virtio_mem *vm,
unsigned long mb_id)
{
const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb;
return find_next_zero_bit(vm->sb_bitmap, bit + vm->nb_sb_per_mb, bit) -
bit;
}
/*
* Prepare the subblock bitmap for the next memory block.
*/
static int virtio_mem_sb_bitmap_prepare_next_mb(struct virtio_mem *vm)
{
const unsigned long old_nb_mb = vm->next_mb_id - vm->first_mb_id;
const unsigned long old_nb_bits = old_nb_mb * vm->nb_sb_per_mb;
const unsigned long new_nb_bits = (old_nb_mb + 1) * vm->nb_sb_per_mb;
int old_pages = PFN_UP(BITS_TO_LONGS(old_nb_bits) * sizeof(long));
int new_pages = PFN_UP(BITS_TO_LONGS(new_nb_bits) * sizeof(long));
unsigned long *new_sb_bitmap, *old_sb_bitmap;
if (vm->sb_bitmap && old_pages == new_pages)
return 0;
new_sb_bitmap = vzalloc(new_pages * PAGE_SIZE);
if (!new_sb_bitmap)
return -ENOMEM;
mutex_lock(&vm->hotplug_mutex);
if (new_sb_bitmap)
memcpy(new_sb_bitmap, vm->sb_bitmap, old_pages * PAGE_SIZE);
old_sb_bitmap = vm->sb_bitmap;
vm->sb_bitmap = new_sb_bitmap;
mutex_unlock(&vm->hotplug_mutex);
vfree(old_sb_bitmap);
return 0;
}
/*
* Try to add a memory block to Linux. This will usually only fail
* if out of memory.
*
* Must not be called with the vm->hotplug_mutex held (possible deadlock with
* onlining code).
*
* Will not modify the state of the memory block.
*/
static int virtio_mem_mb_add(struct virtio_mem *vm, unsigned long mb_id)
{
const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
int nid = vm->nid;
if (nid == NUMA_NO_NODE)
nid = memory_add_physaddr_to_nid(addr);
dev_dbg(&vm->vdev->dev, "adding memory block: %lu\n", mb_id);
return add_memory(nid, addr, memory_block_size_bytes());
}
/*
* Try to remove a memory block from Linux. Will only fail if the memory block
* is not offline.
*
* Must not be called with the vm->hotplug_mutex held (possible deadlock with
* onlining code).
*
* Will not modify the state of the memory block.
*/
static int virtio_mem_mb_remove(struct virtio_mem *vm, unsigned long mb_id)
{
const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
int nid = vm->nid;
if (nid == NUMA_NO_NODE)
nid = memory_add_physaddr_to_nid(addr);
dev_dbg(&vm->vdev->dev, "removing memory block: %lu\n", mb_id);
return remove_memory(nid, addr, memory_block_size_bytes());
}
/*
* Try to offline and remove a memory block from Linux.
*
* Must not be called with the vm->hotplug_mutex held (possible deadlock with
* onlining code).
*
* Will not modify the state of the memory block.
*/
static int virtio_mem_mb_offline_and_remove(struct virtio_mem *vm,
unsigned long mb_id)
{
const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
int nid = vm->nid;
if (nid == NUMA_NO_NODE)
nid = memory_add_physaddr_to_nid(addr);
dev_dbg(&vm->vdev->dev, "offlining and removing memory block: %lu\n",
mb_id);
return offline_and_remove_memory(nid, addr, memory_block_size_bytes());
}
/*
* Trigger the workqueue so the device can perform its magic.
*/
static void virtio_mem_retry(struct virtio_mem *vm)
{
unsigned long flags;
spin_lock_irqsave(&vm->removal_lock, flags);
if (!vm->removing)
queue_work(system_freezable_wq, &vm->wq);
spin_unlock_irqrestore(&vm->removal_lock, flags);
}
static int virtio_mem_translate_node_id(struct virtio_mem *vm, uint16_t node_id)
{
int node = NUMA_NO_NODE;
#if defined(CONFIG_ACPI_NUMA)
if (virtio_has_feature(vm->vdev, VIRTIO_MEM_F_ACPI_PXM))
node = pxm_to_node(node_id);
#endif
return node;
}
/*
* Test if a virtio-mem device overlaps with the given range. Can be called
* from (notifier) callbacks lockless.
*/
static bool virtio_mem_overlaps_range(struct virtio_mem *vm,
unsigned long start, unsigned long size)
{
unsigned long dev_start = virtio_mem_mb_id_to_phys(vm->first_mb_id);
unsigned long dev_end = virtio_mem_mb_id_to_phys(vm->last_mb_id) +
memory_block_size_bytes();
return start < dev_end && dev_start < start + size;
}
/*
* Test if a virtio-mem device owns a memory block. Can be called from
* (notifier) callbacks lockless.
*/
static bool virtio_mem_owned_mb(struct virtio_mem *vm, unsigned long mb_id)
{
return mb_id >= vm->first_mb_id && mb_id <= vm->last_mb_id;
}
static int virtio_mem_notify_going_online(struct virtio_mem *vm,
unsigned long mb_id,
enum zone_type zone)
{
switch (virtio_mem_mb_get_state(vm, mb_id)) {
case VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL:
/*
* We won't allow to online a partially plugged memory block
* to the MOVABLE zone - it would contain unmovable parts.
*/
if (zone == ZONE_MOVABLE) {
dev_warn_ratelimited(&vm->vdev->dev,
"memory block has holes, MOVABLE not supported\n");
return NOTIFY_BAD;
}
return NOTIFY_OK;
case VIRTIO_MEM_MB_STATE_OFFLINE:
return NOTIFY_OK;
default:
break;
}
dev_warn_ratelimited(&vm->vdev->dev,
"memory block onlining denied\n");
return NOTIFY_BAD;
}
static void virtio_mem_notify_offline(struct virtio_mem *vm,
unsigned long mb_id)
{
switch (virtio_mem_mb_get_state(vm, mb_id)) {
case VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL:
virtio_mem_mb_set_state(vm, mb_id,
VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL);
break;
case VIRTIO_MEM_MB_STATE_ONLINE:
case VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE:
virtio_mem_mb_set_state(vm, mb_id,
VIRTIO_MEM_MB_STATE_OFFLINE);
break;
default:
BUG();
break;
}
/*
* Trigger the workqueue, maybe we can now unplug memory. Also,
* when we offline and remove a memory block, this will re-trigger
* us immediately - which is often nice because the removal of
* the memory block (e.g., memmap) might have freed up memory
* on other memory blocks we manage.
*/
virtio_mem_retry(vm);
}
static void virtio_mem_notify_online(struct virtio_mem *vm, unsigned long mb_id,
enum zone_type zone)
{
unsigned long nb_offline;
switch (virtio_mem_mb_get_state(vm, mb_id)) {
case VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL:
BUG_ON(zone == ZONE_MOVABLE);
virtio_mem_mb_set_state(vm, mb_id,
VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL);
break;
case VIRTIO_MEM_MB_STATE_OFFLINE:
if (zone == ZONE_MOVABLE)
virtio_mem_mb_set_state(vm, mb_id,
VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE);
else
virtio_mem_mb_set_state(vm, mb_id,
VIRTIO_MEM_MB_STATE_ONLINE);
break;
default:
BUG();
break;
}
nb_offline = vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] +
vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL];
/* see if we can add new blocks now that we onlined one block */
if (nb_offline == VIRTIO_MEM_NB_OFFLINE_THRESHOLD - 1)
virtio_mem_retry(vm);
}
static void virtio_mem_notify_going_offline(struct virtio_mem *vm,
unsigned long mb_id)
{
const unsigned long nr_pages = PFN_DOWN(vm->subblock_size);
struct page *page;
unsigned long pfn;
int sb_id, i;
for (sb_id = 0; sb_id < vm->nb_sb_per_mb; sb_id++) {
if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
continue;
/*
* Drop our reference to the pages so the memory can get
* offlined and add the unplugged pages to the managed
* page counters (so offlining code can correctly subtract
* them again).
*/
pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
sb_id * vm->subblock_size);
adjust_managed_page_count(pfn_to_page(pfn), nr_pages);
for (i = 0; i < nr_pages; i++) {
page = pfn_to_page(pfn + i);
if (WARN_ON(!page_ref_dec_and_test(page)))
dump_page(page, "unplugged page referenced");
}
}
}
static void virtio_mem_notify_cancel_offline(struct virtio_mem *vm,
unsigned long mb_id)
{
const unsigned long nr_pages = PFN_DOWN(vm->subblock_size);
unsigned long pfn;
int sb_id, i;
for (sb_id = 0; sb_id < vm->nb_sb_per_mb; sb_id++) {
if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
continue;
/*
* Get the reference we dropped when going offline and
* subtract the unplugged pages from the managed page
* counters.
*/
pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
sb_id * vm->subblock_size);
adjust_managed_page_count(pfn_to_page(pfn), -nr_pages);
for (i = 0; i < nr_pages; i++)
page_ref_inc(pfn_to_page(pfn + i));
}
}
/*
* This callback will either be called synchronously from add_memory() or
* asynchronously (e.g., triggered via user space). We have to be careful
* with locking when calling add_memory().
*/
static int virtio_mem_memory_notifier_cb(struct notifier_block *nb,
unsigned long action, void *arg)
{
struct virtio_mem *vm = container_of(nb, struct virtio_mem,
memory_notifier);
struct memory_notify *mhp = arg;
const unsigned long start = PFN_PHYS(mhp->start_pfn);
const unsigned long size = PFN_PHYS(mhp->nr_pages);
const unsigned long mb_id = virtio_mem_phys_to_mb_id(start);
enum zone_type zone;
int rc = NOTIFY_OK;
if (!virtio_mem_overlaps_range(vm, start, size))
return NOTIFY_DONE;
/*
* Memory is onlined/offlined in memory block granularity. We cannot
* cross virtio-mem device boundaries and memory block boundaries. Bail
* out if this ever changes.
*/
if (WARN_ON_ONCE(size != memory_block_size_bytes() ||
!IS_ALIGNED(start, memory_block_size_bytes())))
return NOTIFY_BAD;
/*
* Avoid circular locking lockdep warnings. We lock the mutex
* e.g., in MEM_GOING_ONLINE and unlock it in MEM_ONLINE. The
* blocking_notifier_call_chain() has it's own lock, which gets unlocked
* between both notifier calls and will bail out. False positive.
*/
lockdep_off();
switch (action) {
case MEM_GOING_OFFLINE:
mutex_lock(&vm->hotplug_mutex);
if (vm->removing) {
rc = notifier_from_errno(-EBUSY);
mutex_unlock(&vm->hotplug_mutex);
break;
}
vm->hotplug_active = true;
virtio_mem_notify_going_offline(vm, mb_id);
break;
case MEM_GOING_ONLINE:
mutex_lock(&vm->hotplug_mutex);
if (vm->removing) {
rc = notifier_from_errno(-EBUSY);
mutex_unlock(&vm->hotplug_mutex);
break;
}
vm->hotplug_active = true;
zone = page_zonenum(pfn_to_page(mhp->start_pfn));
rc = virtio_mem_notify_going_online(vm, mb_id, zone);
break;
case MEM_OFFLINE:
virtio_mem_notify_offline(vm, mb_id);
vm->hotplug_active = false;
mutex_unlock(&vm->hotplug_mutex);
break;
case MEM_ONLINE:
zone = page_zonenum(pfn_to_page(mhp->start_pfn));
virtio_mem_notify_online(vm, mb_id, zone);
vm->hotplug_active = false;
mutex_unlock(&vm->hotplug_mutex);
break;
case MEM_CANCEL_OFFLINE:
if (!vm->hotplug_active)
break;
virtio_mem_notify_cancel_offline(vm, mb_id);
vm->hotplug_active = false;
mutex_unlock(&vm->hotplug_mutex);
break;
case MEM_CANCEL_ONLINE:
if (!vm->hotplug_active)
break;
vm->hotplug_active = false;
mutex_unlock(&vm->hotplug_mutex);
break;
default:
break;
}
lockdep_on();
return rc;
}
/*
* Set a range of pages PG_offline. Remember pages that were never onlined
* (via generic_online_page()) using PageDirty().
*/
static void virtio_mem_set_fake_offline(unsigned long pfn,
unsigned int nr_pages, bool onlined)
{
for (; nr_pages--; pfn++) {
struct page *page = pfn_to_page(pfn);
__SetPageOffline(page);
if (!onlined) {
SetPageDirty(page);
/* FIXME: remove after cleanups */
ClearPageReserved(page);
}
}
}
/*
* Clear PG_offline from a range of pages. If the pages were never onlined,
* (via generic_online_page()), clear PageDirty().
*/
static void virtio_mem_clear_fake_offline(unsigned long pfn,
unsigned int nr_pages, bool onlined)
{
for (; nr_pages--; pfn++) {
struct page *page = pfn_to_page(pfn);
__ClearPageOffline(page);
if (!onlined)
ClearPageDirty(page);
}
}
/*
* Release a range of fake-offline pages to the buddy, effectively
* fake-onlining them.
*/
static void virtio_mem_fake_online(unsigned long pfn, unsigned int nr_pages)
{
const int order = MAX_ORDER - 1;
int i;
/*
* We are always called with subblock granularity, which is at least
* aligned to MAX_ORDER - 1.
*/
for (i = 0; i < nr_pages; i += 1 << order) {
struct page *page = pfn_to_page(pfn + i);
/*
* If the page is PageDirty(), it was kept fake-offline when
* onlining the memory block. Otherwise, it was allocated
* using alloc_contig_range(). All pages in a subblock are
* alike.
*/
if (PageDirty(page)) {
virtio_mem_clear_fake_offline(pfn + i, 1 << order,
false);
generic_online_page(page, order);
} else {
virtio_mem_clear_fake_offline(pfn + i, 1 << order,
true);
free_contig_range(pfn + i, 1 << order);
adjust_managed_page_count(page, 1 << order);
}
}
}
static void virtio_mem_online_page_cb(struct page *page, unsigned int order)
{
const unsigned long addr = page_to_phys(page);
const unsigned long mb_id = virtio_mem_phys_to_mb_id(addr);
struct virtio_mem *vm;
int sb_id;
/*
* We exploit here that subblocks have at least MAX_ORDER - 1
* size/alignment and that this callback is is called with such a
* size/alignment. So we cannot cross subblocks and therefore
* also not memory blocks.
*/
rcu_read_lock();
list_for_each_entry_rcu(vm, &virtio_mem_devices, next) {
if (!virtio_mem_owned_mb(vm, mb_id))
continue;
sb_id = virtio_mem_phys_to_sb_id(vm, addr);
/*
* If plugged, online the pages, otherwise, set them fake
* offline (PageOffline).
*/
if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
generic_online_page(page, order);
else
virtio_mem_set_fake_offline(PFN_DOWN(addr), 1 << order,
false);
rcu_read_unlock();
return;
}
rcu_read_unlock();
/* not virtio-mem memory, but e.g., a DIMM. online it */
generic_online_page(page, order);
}
static uint64_t virtio_mem_send_request(struct virtio_mem *vm,
const struct virtio_mem_req *req)
{
struct scatterlist *sgs[2], sg_req, sg_resp;
unsigned int len;
int rc;
/* don't use the request residing on the stack (vaddr) */
vm->req = *req;
/* out: buffer for request */
sg_init_one(&sg_req, &vm->req, sizeof(vm->req));
sgs[0] = &sg_req;
/* in: buffer for response */
sg_init_one(&sg_resp, &vm->resp, sizeof(vm->resp));
sgs[1] = &sg_resp;
rc = virtqueue_add_sgs(vm->vq, sgs, 1, 1, vm, GFP_KERNEL);
if (rc < 0)
return rc;
virtqueue_kick(vm->vq);
/* wait for a response */
wait_event(vm->host_resp, virtqueue_get_buf(vm->vq, &len));
return virtio16_to_cpu(vm->vdev, vm->resp.type);
}
static int virtio_mem_send_plug_request(struct virtio_mem *vm, uint64_t addr,
uint64_t size)
{
const uint64_t nb_vm_blocks = size / vm->device_block_size;
const struct virtio_mem_req req = {
.type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_PLUG),
.u.plug.addr = cpu_to_virtio64(vm->vdev, addr),
.u.plug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
};
if (atomic_read(&vm->config_changed))
return -EAGAIN;
switch (virtio_mem_send_request(vm, &req)) {
case VIRTIO_MEM_RESP_ACK:
vm->plugged_size += size;
return 0;
case VIRTIO_MEM_RESP_NACK:
return -EAGAIN;
case VIRTIO_MEM_RESP_BUSY:
return -ETXTBSY;
case VIRTIO_MEM_RESP_ERROR:
return -EINVAL;
default:
return -ENOMEM;
}
}
static int virtio_mem_send_unplug_request(struct virtio_mem *vm, uint64_t addr,
uint64_t size)
{
const uint64_t nb_vm_blocks = size / vm->device_block_size;
const struct virtio_mem_req req = {
.type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG),
.u.unplug.addr = cpu_to_virtio64(vm->vdev, addr),
.u.unplug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
};
if (atomic_read(&vm->config_changed))
return -EAGAIN;
switch (virtio_mem_send_request(vm, &req)) {
case VIRTIO_MEM_RESP_ACK:
vm->plugged_size -= size;
return 0;
case VIRTIO_MEM_RESP_BUSY:
return -ETXTBSY;
case VIRTIO_MEM_RESP_ERROR:
return -EINVAL;
default:
return -ENOMEM;
}
}
static int virtio_mem_send_unplug_all_request(struct virtio_mem *vm)
{
const struct virtio_mem_req req = {
.type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG_ALL),
};
switch (virtio_mem_send_request(vm, &req)) {
case VIRTIO_MEM_RESP_ACK:
vm->unplug_all_required = false;
vm->plugged_size = 0;
/* usable region might have shrunk */
atomic_set(&vm->config_changed, 1);
return 0;
case VIRTIO_MEM_RESP_BUSY:
return -ETXTBSY;
default:
return -ENOMEM;
}
}
/*
* Plug selected subblocks. Updates the plugged state, but not the state
* of the memory block.
*/
static int virtio_mem_mb_plug_sb(struct virtio_mem *vm, unsigned long mb_id,
int sb_id, int count)
{
const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) +
sb_id * vm->subblock_size;
const uint64_t size = count * vm->subblock_size;
int rc;
dev_dbg(&vm->vdev->dev, "plugging memory block: %lu : %i - %i\n", mb_id,
sb_id, sb_id + count - 1);
rc = virtio_mem_send_plug_request(vm, addr, size);
if (!rc)
virtio_mem_mb_set_sb_plugged(vm, mb_id, sb_id, count);
return rc;
}
/*
* Unplug selected subblocks. Updates the plugged state, but not the state
* of the memory block.
*/
static int virtio_mem_mb_unplug_sb(struct virtio_mem *vm, unsigned long mb_id,
int sb_id, int count)
{
const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) +
sb_id * vm->subblock_size;
const uint64_t size = count * vm->subblock_size;
int rc;
dev_dbg(&vm->vdev->dev, "unplugging memory block: %lu : %i - %i\n",
mb_id, sb_id, sb_id + count - 1);
rc = virtio_mem_send_unplug_request(vm, addr, size);
if (!rc)
virtio_mem_mb_set_sb_unplugged(vm, mb_id, sb_id, count);
return rc;
}
/*
* Unplug the desired number of plugged subblocks of a offline or not-added
* memory block. Will fail if any subblock cannot get unplugged (instead of
* skipping it).
*
* Will not modify the state of the memory block.
*
* Note: can fail after some subblocks were unplugged.
*/
static int virtio_mem_mb_unplug_any_sb(struct virtio_mem *vm,
unsigned long mb_id, uint64_t *nb_sb)
{
int sb_id, count;
int rc;
sb_id = vm->nb_sb_per_mb - 1;
while (*nb_sb) {
/* Find the next candidate subblock */
while (sb_id >= 0 &&
virtio_mem_mb_test_sb_unplugged(vm, mb_id, sb_id, 1))
sb_id--;
if (sb_id < 0)
break;
/* Try to unplug multiple subblocks at a time */
count = 1;
while (count < *nb_sb && sb_id > 0 &&
virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id - 1, 1)) {
count++;
sb_id--;
}
rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, count);
if (rc)
return rc;
*nb_sb -= count;
sb_id--;
}
return 0;
}
/*
* Unplug all plugged subblocks of an offline or not-added memory block.
*
* Will not modify the state of the memory block.
*
* Note: can fail after some subblocks were unplugged.
*/
static int virtio_mem_mb_unplug(struct virtio_mem *vm, unsigned long mb_id)
{
uint64_t nb_sb = vm->nb_sb_per_mb;
return virtio_mem_mb_unplug_any_sb(vm, mb_id, &nb_sb);
}
/*
* Prepare tracking data for the next memory block.
*/
static int virtio_mem_prepare_next_mb(struct virtio_mem *vm,
unsigned long *mb_id)
{
int rc;
if (vm->next_mb_id > vm->last_usable_mb_id)
return -ENOSPC;
/* Resize the state array if required. */
rc = virtio_mem_mb_state_prepare_next_mb(vm);
if (rc)
return rc;
/* Resize the subblock bitmap if required. */
rc = virtio_mem_sb_bitmap_prepare_next_mb(vm);
if (rc)
return rc;
vm->nb_mb_state[VIRTIO_MEM_MB_STATE_UNUSED]++;
*mb_id = vm->next_mb_id++;
return 0;
}
/*
* Don't add too many blocks that are not onlined yet to avoid running OOM.
*/
static bool virtio_mem_too_many_mb_offline(struct virtio_mem *vm)
{
unsigned long nb_offline;
nb_offline = vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] +
vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL];
return nb_offline >= VIRTIO_MEM_NB_OFFLINE_THRESHOLD;
}
/*
* Try to plug the desired number of subblocks and add the memory block
* to Linux.
*
* Will modify the state of the memory block.
*/
static int virtio_mem_mb_plug_and_add(struct virtio_mem *vm,
unsigned long mb_id,
uint64_t *nb_sb)
{
const int count = min_t(int, *nb_sb, vm->nb_sb_per_mb);
int rc, rc2;
if (WARN_ON_ONCE(!count))
return -EINVAL;
/*
* Plug the requested number of subblocks before adding it to linux,
* so that onlining will directly online all plugged subblocks.
*/
rc = virtio_mem_mb_plug_sb(vm, mb_id, 0, count);
if (rc)
return rc;
/*
* Mark the block properly offline before adding it to Linux,
* so the memory notifiers will find the block in the right state.
*/
if (count == vm->nb_sb_per_mb)
virtio_mem_mb_set_state(vm, mb_id,
VIRTIO_MEM_MB_STATE_OFFLINE);
else
virtio_mem_mb_set_state(vm, mb_id,
VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL);
/* Add the memory block to linux - if that fails, try to unplug. */
rc = virtio_mem_mb_add(vm, mb_id);
if (rc) {
enum virtio_mem_mb_state new_state = VIRTIO_MEM_MB_STATE_UNUSED;
dev_err(&vm->vdev->dev,
"adding memory block %lu failed with %d\n", mb_id, rc);
rc2 = virtio_mem_mb_unplug_sb(vm, mb_id, 0, count);
/*
* TODO: Linux MM does not properly clean up yet in all cases
* where adding of memory failed - especially on -ENOMEM.
*/
if (rc2)
new_state = VIRTIO_MEM_MB_STATE_PLUGGED;
virtio_mem_mb_set_state(vm, mb_id, new_state);
return rc;
}
*nb_sb -= count;
return 0;
}
/*
* Try to plug the desired number of subblocks of a memory block that
* is already added to Linux.
*
* Will modify the state of the memory block.
*
* Note: Can fail after some subblocks were successfully plugged.
*/
static int virtio_mem_mb_plug_any_sb(struct virtio_mem *vm, unsigned long mb_id,
uint64_t *nb_sb, bool online)
{
unsigned long pfn, nr_pages;
int sb_id, count;
int rc;
if (WARN_ON_ONCE(!*nb_sb))
return -EINVAL;
while (*nb_sb) {
sb_id = virtio_mem_mb_first_unplugged_sb(vm, mb_id);
if (sb_id >= vm->nb_sb_per_mb)
break;
count = 1;
while (count < *nb_sb &&
sb_id + count < vm->nb_sb_per_mb &&
!virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id + count,
1))
count++;
rc = virtio_mem_mb_plug_sb(vm, mb_id, sb_id, count);
if (rc)
return rc;
*nb_sb -= count;
if (!online)
continue;
/* fake-online the pages if the memory block is online */
pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
sb_id * vm->subblock_size);
nr_pages = PFN_DOWN(count * vm->subblock_size);
virtio_mem_fake_online(pfn, nr_pages);
}
if (virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
if (online)
virtio_mem_mb_set_state(vm, mb_id,
VIRTIO_MEM_MB_STATE_ONLINE);
else
virtio_mem_mb_set_state(vm, mb_id,
VIRTIO_MEM_MB_STATE_OFFLINE);
}
return rc;
}
/*
* Try to plug the requested amount of memory.
*/
static int virtio_mem_plug_request(struct virtio_mem *vm, uint64_t diff)
{
uint64_t nb_sb = diff / vm->subblock_size;
unsigned long mb_id;
int rc;
if (!nb_sb)
return 0;
/* Don't race with onlining/offlining */
mutex_lock(&vm->hotplug_mutex);
/* Try to plug subblocks of partially plugged online blocks. */
virtio_mem_for_each_mb_state(vm, mb_id,
VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL) {
rc = virtio_mem_mb_plug_any_sb(vm, mb_id, &nb_sb, true);
if (rc || !nb_sb)
goto out_unlock;
cond_resched();
}
/* Try to plug subblocks of partially plugged offline blocks. */
virtio_mem_for_each_mb_state(vm, mb_id,
VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) {
rc = virtio_mem_mb_plug_any_sb(vm, mb_id, &nb_sb, false);
if (rc || !nb_sb)
goto out_unlock;
cond_resched();
}
/*
* We won't be working on online/offline memory blocks from this point,
* so we can't race with memory onlining/offlining. Drop the mutex.
*/
mutex_unlock(&vm->hotplug_mutex);
/* Try to plug and add unused blocks */
virtio_mem_for_each_mb_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED) {
if (virtio_mem_too_many_mb_offline(vm))
return -ENOSPC;
rc = virtio_mem_mb_plug_and_add(vm, mb_id, &nb_sb);
if (rc || !nb_sb)
return rc;
cond_resched();
}
/* Try to prepare, plug and add new blocks */
while (nb_sb) {
if (virtio_mem_too_many_mb_offline(vm))
return -ENOSPC;
rc = virtio_mem_prepare_next_mb(vm, &mb_id);
if (rc)
return rc;
rc = virtio_mem_mb_plug_and_add(vm, mb_id, &nb_sb);
if (rc)
return rc;
cond_resched();
}
return 0;
out_unlock:
mutex_unlock(&vm->hotplug_mutex);
return rc;
}
/*
* Unplug the desired number of plugged subblocks of an offline memory block.
* Will fail if any subblock cannot get unplugged (instead of skipping it).
*
* Will modify the state of the memory block. Might temporarily drop the
* hotplug_mutex.
*
* Note: Can fail after some subblocks were successfully unplugged.
*/
static int virtio_mem_mb_unplug_any_sb_offline(struct virtio_mem *vm,
unsigned long mb_id,
uint64_t *nb_sb)
{
int rc;
rc = virtio_mem_mb_unplug_any_sb(vm, mb_id, nb_sb);
/* some subblocks might have been unplugged even on failure */
if (!virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb))
virtio_mem_mb_set_state(vm, mb_id,
VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL);
if (rc)
return rc;
if (virtio_mem_mb_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
/*
* Remove the block from Linux - this should never fail.
* Hinder the block from getting onlined by marking it
* unplugged. Temporarily drop the mutex, so
* any pending GOING_ONLINE requests can be serviced/rejected.
*/
virtio_mem_mb_set_state(vm, mb_id,
VIRTIO_MEM_MB_STATE_UNUSED);
mutex_unlock(&vm->hotplug_mutex);
rc = virtio_mem_mb_remove(vm, mb_id);
BUG_ON(rc);
mutex_lock(&vm->hotplug_mutex);
}
return 0;
}
/*
* Unplug the given plugged subblocks of an online memory block.
*
* Will modify the state of the memory block.
*/
static int virtio_mem_mb_unplug_sb_online(struct virtio_mem *vm,
unsigned long mb_id, int sb_id,
int count)
{
const unsigned long nr_pages = PFN_DOWN(vm->subblock_size) * count;
unsigned long start_pfn;
int rc;
start_pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
sb_id * vm->subblock_size);
rc = alloc_contig_range(start_pfn, start_pfn + nr_pages,
MIGRATE_MOVABLE, GFP_KERNEL);
if (rc == -ENOMEM)
/* whoops, out of memory */
return rc;
if (rc)
return -EBUSY;
/* Mark it as fake-offline before unplugging it */
virtio_mem_set_fake_offline(start_pfn, nr_pages, true);
adjust_managed_page_count(pfn_to_page(start_pfn), -nr_pages);
/* Try to unplug the allocated memory */
rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, count);
if (rc) {
/* Return the memory to the buddy. */
virtio_mem_fake_online(start_pfn, nr_pages);
return rc;
}
virtio_mem_mb_set_state(vm, mb_id,
VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL);
return 0;
}
/*
* Unplug the desired number of plugged subblocks of an online memory block.
* Will skip subblock that are busy.
*
* Will modify the state of the memory block. Might temporarily drop the
* hotplug_mutex.
*
* Note: Can fail after some subblocks were successfully unplugged. Can
* return 0 even if subblocks were busy and could not get unplugged.
*/
static int virtio_mem_mb_unplug_any_sb_online(struct virtio_mem *vm,
unsigned long mb_id,
uint64_t *nb_sb)
{
int rc, sb_id;
/* If possible, try to unplug the complete block in one shot. */
if (*nb_sb >= vm->nb_sb_per_mb &&
virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
rc = virtio_mem_mb_unplug_sb_online(vm, mb_id, 0,
vm->nb_sb_per_mb);
if (!rc) {
*nb_sb -= vm->nb_sb_per_mb;
goto unplugged;
} else if (rc != -EBUSY)
return rc;
}
/* Fallback to single subblocks. */
for (sb_id = vm->nb_sb_per_mb - 1; sb_id >= 0 && *nb_sb; sb_id--) {
/* Find the next candidate subblock */
while (sb_id >= 0 &&
!virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
sb_id--;
if (sb_id < 0)
break;
rc = virtio_mem_mb_unplug_sb_online(vm, mb_id, sb_id, 1);
if (rc == -EBUSY)
continue;
else if (rc)
return rc;
*nb_sb -= 1;
}
unplugged:
/*
* Once all subblocks of a memory block were unplugged, offline and
* remove it. This will usually not fail, as no memory is in use
* anymore - however some other notifiers might NACK the request.
*/
if (virtio_mem_mb_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
mutex_unlock(&vm->hotplug_mutex);
rc = virtio_mem_mb_offline_and_remove(vm, mb_id);
mutex_lock(&vm->hotplug_mutex);
if (!rc)
virtio_mem_mb_set_state(vm, mb_id,
VIRTIO_MEM_MB_STATE_UNUSED);
}
return 0;
}
/*
* Try to unplug the requested amount of memory.
*/
static int virtio_mem_unplug_request(struct virtio_mem *vm, uint64_t diff)
{
uint64_t nb_sb = diff / vm->subblock_size;
unsigned long mb_id;
int rc;
if (!nb_sb)
return 0;
/*
* We'll drop the mutex a couple of times when it is safe to do so.
* This might result in some blocks switching the state (online/offline)
* and we could miss them in this run - we will retry again later.
*/
mutex_lock(&vm->hotplug_mutex);
/* Try to unplug subblocks of partially plugged offline blocks. */
virtio_mem_for_each_mb_state_rev(vm, mb_id,
VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) {
rc = virtio_mem_mb_unplug_any_sb_offline(vm, mb_id,
&nb_sb);
if (rc || !nb_sb)
goto out_unlock;
cond_resched();
}
/* Try to unplug subblocks of plugged offline blocks. */
virtio_mem_for_each_mb_state_rev(vm, mb_id,
VIRTIO_MEM_MB_STATE_OFFLINE) {
rc = virtio_mem_mb_unplug_any_sb_offline(vm, mb_id,
&nb_sb);
if (rc || !nb_sb)
goto out_unlock;
cond_resched();
}
if (!unplug_online) {
mutex_unlock(&vm->hotplug_mutex);
return 0;
}
/* Try to unplug subblocks of partially plugged online blocks. */
virtio_mem_for_each_mb_state_rev(vm, mb_id,
VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL) {
rc = virtio_mem_mb_unplug_any_sb_online(vm, mb_id,
&nb_sb);
if (rc || !nb_sb)
goto out_unlock;
mutex_unlock(&vm->hotplug_mutex);
cond_resched();
mutex_lock(&vm->hotplug_mutex);
}
/* Try to unplug subblocks of plugged online blocks. */
virtio_mem_for_each_mb_state_rev(vm, mb_id,
VIRTIO_MEM_MB_STATE_ONLINE) {
rc = virtio_mem_mb_unplug_any_sb_online(vm, mb_id,
&nb_sb);
if (rc || !nb_sb)
goto out_unlock;
mutex_unlock(&vm->hotplug_mutex);
cond_resched();
mutex_lock(&vm->hotplug_mutex);
}
mutex_unlock(&vm->hotplug_mutex);
return nb_sb ? -EBUSY : 0;
out_unlock:
mutex_unlock(&vm->hotplug_mutex);
return rc;
}
/*
* Try to unplug all blocks that couldn't be unplugged before, for example,
* because the hypervisor was busy.
*/
static int virtio_mem_unplug_pending_mb(struct virtio_mem *vm)
{
unsigned long mb_id;
int rc;
virtio_mem_for_each_mb_state(vm, mb_id, VIRTIO_MEM_MB_STATE_PLUGGED) {
rc = virtio_mem_mb_unplug(vm, mb_id);
if (rc)
return rc;
virtio_mem_mb_set_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED);
}
return 0;
}
/*
* Update all parts of the config that could have changed.
*/
static void virtio_mem_refresh_config(struct virtio_mem *vm)
{
const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS;
uint64_t new_plugged_size, usable_region_size, end_addr;
/* the plugged_size is just a reflection of what _we_ did previously */
virtio_cread(vm->vdev, struct virtio_mem_config, plugged_size,
&new_plugged_size);
if (WARN_ON_ONCE(new_plugged_size != vm->plugged_size))
vm->plugged_size = new_plugged_size;
/* calculate the last usable memory block id */
virtio_cread(vm->vdev, struct virtio_mem_config,
usable_region_size, &usable_region_size);
end_addr = vm->addr + usable_region_size;
end_addr = min(end_addr, phys_limit);
vm->last_usable_mb_id = virtio_mem_phys_to_mb_id(end_addr) - 1;
/* see if there is a request to change the size */
virtio_cread(vm->vdev, struct virtio_mem_config, requested_size,
&vm->requested_size);
dev_info(&vm->vdev->dev, "plugged size: 0x%llx", vm->plugged_size);
dev_info(&vm->vdev->dev, "requested size: 0x%llx", vm->requested_size);
}
/*
* Workqueue function for handling plug/unplug requests and config updates.
*/
static void virtio_mem_run_wq(struct work_struct *work)
{
struct virtio_mem *vm = container_of(work, struct virtio_mem, wq);
uint64_t diff;
int rc;
hrtimer_cancel(&vm->retry_timer);
if (vm->broken)
return;
retry:
rc = 0;
/* Make sure we start with a clean state if there are leftovers. */
if (unlikely(vm->unplug_all_required))
rc = virtio_mem_send_unplug_all_request(vm);
if (atomic_read(&vm->config_changed)) {
atomic_set(&vm->config_changed, 0);
virtio_mem_refresh_config(vm);
}
/* Unplug any leftovers from previous runs */
if (!rc)
rc = virtio_mem_unplug_pending_mb(vm);
if (!rc && vm->requested_size != vm->plugged_size) {
if (vm->requested_size > vm->plugged_size) {
diff = vm->requested_size - vm->plugged_size;
rc = virtio_mem_plug_request(vm, diff);
} else {
diff = vm->plugged_size - vm->requested_size;
rc = virtio_mem_unplug_request(vm, diff);
}
}
switch (rc) {
case 0:
vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS;
break;
case -ENOSPC:
/*
* We cannot add any more memory (alignment, physical limit)
* or we have too many offline memory blocks.
*/
break;
case -ETXTBSY:
/*
* The hypervisor cannot process our request right now
* (e.g., out of memory, migrating);
*/
case -EBUSY:
/*
* We cannot free up any memory to unplug it (all plugged memory
* is busy).
*/
case -ENOMEM:
/* Out of memory, try again later. */
hrtimer_start(&vm->retry_timer, ms_to_ktime(vm->retry_timer_ms),
HRTIMER_MODE_REL);
break;
case -EAGAIN:
/* Retry immediately (e.g., the config changed). */
goto retry;
default:
/* Unknown error, mark as broken */
dev_err(&vm->vdev->dev,
"unknown error, marking device broken: %d\n", rc);
vm->broken = true;
}
}
static enum hrtimer_restart virtio_mem_timer_expired(struct hrtimer *timer)
{
struct virtio_mem *vm = container_of(timer, struct virtio_mem,
retry_timer);
virtio_mem_retry(vm);
vm->retry_timer_ms = min_t(unsigned int, vm->retry_timer_ms * 2,
VIRTIO_MEM_RETRY_TIMER_MAX_MS);
return HRTIMER_NORESTART;
}
static void virtio_mem_handle_response(struct virtqueue *vq)
{
struct virtio_mem *vm = vq->vdev->priv;
wake_up(&vm->host_resp);
}
static int virtio_mem_init_vq(struct virtio_mem *vm)
{
struct virtqueue *vq;
vq = virtio_find_single_vq(vm->vdev, virtio_mem_handle_response,
"guest-request");
if (IS_ERR(vq))
return PTR_ERR(vq);
vm->vq = vq;
return 0;
}
static int virtio_mem_init(struct virtio_mem *vm)
{
const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS;
uint16_t node_id;
if (!vm->vdev->config->get) {
dev_err(&vm->vdev->dev, "config access disabled\n");
return -EINVAL;
}
/*
* We don't want to (un)plug or reuse any memory when in kdump. The
* memory is still accessible (but not mapped).
*/
if (is_kdump_kernel()) {
dev_warn(&vm->vdev->dev, "disabled in kdump kernel\n");
return -EBUSY;
}
/* Fetch all properties that can't change. */
virtio_cread(vm->vdev, struct virtio_mem_config, plugged_size,
&vm->plugged_size);
virtio_cread(vm->vdev, struct virtio_mem_config, block_size,
&vm->device_block_size);
virtio_cread(vm->vdev, struct virtio_mem_config, node_id,
&node_id);
vm->nid = virtio_mem_translate_node_id(vm, node_id);
virtio_cread(vm->vdev, struct virtio_mem_config, addr, &vm->addr);
virtio_cread(vm->vdev, struct virtio_mem_config, region_size,
&vm->region_size);
/*
* We always hotplug memory in memory block granularity. This way,
* we have to wait for exactly one memory block to online.
*/
if (vm->device_block_size > memory_block_size_bytes()) {
dev_err(&vm->vdev->dev,
"The block size is not supported (too big).\n");
return -EINVAL;
}
/* bad device setup - warn only */
if (!IS_ALIGNED(vm->addr, memory_block_size_bytes()))
dev_warn(&vm->vdev->dev,
"The alignment of the physical start address can make some memory unusable.\n");
if (!IS_ALIGNED(vm->addr + vm->region_size, memory_block_size_bytes()))
dev_warn(&vm->vdev->dev,
"The alignment of the physical end address can make some memory unusable.\n");
if (vm->addr + vm->region_size > phys_limit)
dev_warn(&vm->vdev->dev,
"Some memory is not addressable. This can make some memory unusable.\n");
/*
* Calculate the subblock size:
* - At least MAX_ORDER - 1 / pageblock_order.
* - At least the device block size.
* In the worst case, a single subblock per memory block.
*/
vm->subblock_size = PAGE_SIZE * 1ul << max_t(uint32_t, MAX_ORDER - 1,
pageblock_order);
vm->subblock_size = max_t(uint64_t, vm->device_block_size,
vm->subblock_size);
vm->nb_sb_per_mb = memory_block_size_bytes() / vm->subblock_size;
/* Round up to the next full memory block */
vm->first_mb_id = virtio_mem_phys_to_mb_id(vm->addr - 1 +
memory_block_size_bytes());
vm->next_mb_id = vm->first_mb_id;
vm->last_mb_id = virtio_mem_phys_to_mb_id(vm->addr +
vm->region_size) - 1;
dev_info(&vm->vdev->dev, "start address: 0x%llx", vm->addr);
dev_info(&vm->vdev->dev, "region size: 0x%llx", vm->region_size);
dev_info(&vm->vdev->dev, "device block size: 0x%llx",
(unsigned long long)vm->device_block_size);
dev_info(&vm->vdev->dev, "memory block size: 0x%lx",
memory_block_size_bytes());
dev_info(&vm->vdev->dev, "subblock size: 0x%llx",
(unsigned long long)vm->subblock_size);
if (vm->nid != NUMA_NO_NODE)
dev_info(&vm->vdev->dev, "nid: %d", vm->nid);
return 0;
}
static int virtio_mem_create_resource(struct virtio_mem *vm)
{
/*
* When force-unloading the driver and removing the device, we
* could have a garbage pointer. Duplicate the string.
*/
const char *name = kstrdup(dev_name(&vm->vdev->dev), GFP_KERNEL);
if (!name)
return -ENOMEM;
vm->parent_resource = __request_mem_region(vm->addr, vm->region_size,
name, IORESOURCE_SYSTEM_RAM);
if (!vm->parent_resource) {
kfree(name);
dev_warn(&vm->vdev->dev, "could not reserve device region\n");
dev_info(&vm->vdev->dev,
"reloading the driver is not supported\n");
return -EBUSY;
}
/* The memory is not actually busy - make add_memory() work. */
vm->parent_resource->flags &= ~IORESOURCE_BUSY;
return 0;
}
static void virtio_mem_delete_resource(struct virtio_mem *vm)
{
const char *name;
if (!vm->parent_resource)
return;
name = vm->parent_resource->name;
release_resource(vm->parent_resource);
kfree(vm->parent_resource);
kfree(name);
vm->parent_resource = NULL;
}
static int virtio_mem_probe(struct virtio_device *vdev)
{
struct virtio_mem *vm;
int rc;
BUILD_BUG_ON(sizeof(struct virtio_mem_req) != 24);
BUILD_BUG_ON(sizeof(struct virtio_mem_resp) != 10);
vdev->priv = vm = kzalloc(sizeof(*vm), GFP_KERNEL);
if (!vm)
return -ENOMEM;
init_waitqueue_head(&vm->host_resp);
vm->vdev = vdev;
INIT_WORK(&vm->wq, virtio_mem_run_wq);
mutex_init(&vm->hotplug_mutex);
INIT_LIST_HEAD(&vm->next);
spin_lock_init(&vm->removal_lock);
hrtimer_init(&vm->retry_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
vm->retry_timer.function = virtio_mem_timer_expired;
vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS;
/* register the virtqueue */
rc = virtio_mem_init_vq(vm);
if (rc)
goto out_free_vm;
/* initialize the device by querying the config */
rc = virtio_mem_init(vm);
if (rc)
goto out_del_vq;
/* create the parent resource for all memory */
rc = virtio_mem_create_resource(vm);
if (rc)
goto out_del_vq;
/*
* If we still have memory plugged, we have to unplug all memory first.
* Registering our parent resource makes sure that this memory isn't
* actually in use (e.g., trying to reload the driver).
*/
if (vm->plugged_size) {
vm->unplug_all_required = 1;
dev_info(&vm->vdev->dev, "unplugging all memory is required\n");
}
/* register callbacks */
vm->memory_notifier.notifier_call = virtio_mem_memory_notifier_cb;
rc = register_memory_notifier(&vm->memory_notifier);
if (rc)
goto out_del_resource;
rc = register_virtio_mem_device(vm);
if (rc)
goto out_unreg_mem;
virtio_device_ready(vdev);
/* trigger a config update to start processing the requested_size */
atomic_set(&vm->config_changed, 1);
queue_work(system_freezable_wq, &vm->wq);
return 0;
out_unreg_mem:
unregister_memory_notifier(&vm->memory_notifier);
out_del_resource:
virtio_mem_delete_resource(vm);
out_del_vq:
vdev->config->del_vqs(vdev);
out_free_vm:
kfree(vm);
vdev->priv = NULL;
return rc;
}
static void virtio_mem_remove(struct virtio_device *vdev)
{
struct virtio_mem *vm = vdev->priv;
unsigned long mb_id;
int rc;
/*
* Make sure the workqueue won't be triggered anymore and no memory
* blocks can be onlined/offlined until we're finished here.
*/
mutex_lock(&vm->hotplug_mutex);
spin_lock_irq(&vm->removal_lock);
vm->removing = true;
spin_unlock_irq(&vm->removal_lock);
mutex_unlock(&vm->hotplug_mutex);
/* wait until the workqueue stopped */
cancel_work_sync(&vm->wq);
hrtimer_cancel(&vm->retry_timer);
/*
* After we unregistered our callbacks, user space can online partially
* plugged offline blocks. Make sure to remove them.
*/
virtio_mem_for_each_mb_state(vm, mb_id,
VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) {
rc = virtio_mem_mb_remove(vm, mb_id);
BUG_ON(rc);
virtio_mem_mb_set_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED);
}
/*
* After we unregistered our callbacks, user space can no longer
* offline partially plugged online memory blocks. No need to worry
* about them.
*/
/* unregister callbacks */
unregister_virtio_mem_device(vm);
unregister_memory_notifier(&vm->memory_notifier);
/*
* There is no way we could reliably remove all memory we have added to
* the system. And there is no way to stop the driver/device from going
* away. Warn at least.
*/
if (vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] ||
vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL] ||
vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE] ||
vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL] ||
vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE])
dev_warn(&vdev->dev, "device still has system memory added\n");
else
virtio_mem_delete_resource(vm);
/* remove all tracking data - no locking needed */
vfree(vm->mb_state);
vfree(vm->sb_bitmap);
/* reset the device and cleanup the queues */
vdev->config->reset(vdev);
vdev->config->del_vqs(vdev);
kfree(vm);
vdev->priv = NULL;
}
static void virtio_mem_config_changed(struct virtio_device *vdev)
{
struct virtio_mem *vm = vdev->priv;
atomic_set(&vm->config_changed, 1);
virtio_mem_retry(vm);
}
#ifdef CONFIG_PM_SLEEP
static int virtio_mem_freeze(struct virtio_device *vdev)
{
/*
* When restarting the VM, all memory is usually unplugged. Don't
* allow to suspend/hibernate.
*/
dev_err(&vdev->dev, "save/restore not supported.\n");
return -EPERM;
}
static int virtio_mem_restore(struct virtio_device *vdev)
{
return -EPERM;
}
#endif
static unsigned int virtio_mem_features[] = {
#if defined(CONFIG_NUMA) && defined(CONFIG_ACPI_NUMA)
VIRTIO_MEM_F_ACPI_PXM,
#endif
};
static struct virtio_device_id virtio_mem_id_table[] = {
{ VIRTIO_ID_MEM, VIRTIO_DEV_ANY_ID },
{ 0 },
};
static struct virtio_driver virtio_mem_driver = {
.feature_table = virtio_mem_features,
.feature_table_size = ARRAY_SIZE(virtio_mem_features),
.driver.name = KBUILD_MODNAME,
.driver.owner = THIS_MODULE,
.id_table = virtio_mem_id_table,
.probe = virtio_mem_probe,
.remove = virtio_mem_remove,
.config_changed = virtio_mem_config_changed,
#ifdef CONFIG_PM_SLEEP
.freeze = virtio_mem_freeze,
.restore = virtio_mem_restore,
#endif
};
module_virtio_driver(virtio_mem_driver);
MODULE_DEVICE_TABLE(virtio, virtio_mem_id_table);
MODULE_AUTHOR("David Hildenbrand <david@redhat.com>");
MODULE_DESCRIPTION("Virtio-mem driver");
MODULE_LICENSE("GPL");
...@@ -466,10 +466,8 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned nvqs, ...@@ -466,10 +466,8 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned nvqs,
int irq = platform_get_irq(vm_dev->pdev, 0); int irq = platform_get_irq(vm_dev->pdev, 0);
int i, err, queue_idx = 0; int i, err, queue_idx = 0;
if (irq < 0) { if (irq < 0)
dev_err(&vdev->dev, "Cannot get IRQ resource\n");
return irq; return irq;
}
err = request_irq(irq, vm_interrupt, IRQF_SHARED, err = request_irq(irq, vm_interrupt, IRQF_SHARED,
dev_name(&vdev->dev), vm_dev); dev_name(&vdev->dev), vm_dev);
......
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
#include <linux/delay.h> #include <linux/delay.h>
#define VIRTIO_PCI_NO_LEGACY #define VIRTIO_PCI_NO_LEGACY
#define VIRTIO_RING_NO_LEGACY
#include "virtio_pci_common.h" #include "virtio_pci_common.h"
/* /*
......
...@@ -318,6 +318,7 @@ extern void try_offline_node(int nid); ...@@ -318,6 +318,7 @@ extern void try_offline_node(int nid);
extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages); extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages);
extern int remove_memory(int nid, u64 start, u64 size); extern int remove_memory(int nid, u64 start, u64 size);
extern void __remove_memory(int nid, u64 start, u64 size); extern void __remove_memory(int nid, u64 start, u64 size);
extern int offline_and_remove_memory(int nid, u64 start, u64 size);
#else #else
static inline void try_offline_node(int nid) {} static inline void try_offline_node(int nid) {}
......
...@@ -777,6 +777,16 @@ PAGE_TYPE_OPS(Buddy, buddy) ...@@ -777,6 +777,16 @@ PAGE_TYPE_OPS(Buddy, buddy)
* not onlined when onlining the section). * not onlined when onlining the section).
* The content of these pages is effectively stale. Such pages should not * The content of these pages is effectively stale. Such pages should not
* be touched (read/write/dump/save) except by their owner. * be touched (read/write/dump/save) except by their owner.
*
* If a driver wants to allow to offline unmovable PageOffline() pages without
* putting them back to the buddy, it can do so via the memory notifier by
* decrementing the reference count in MEM_GOING_OFFLINE and incrementing the
* reference count in MEM_CANCEL_OFFLINE. When offlining, the PageOffline()
* pages (now with a reference count of zero) are treated like free pages,
* allowing the containing memory block to get offlined. A driver that
* relies on this feature is aware that re-onlining the memory block will
* require to re-set the pages PageOffline() and not giving them to the
* buddy via online_page_callback_t.
*/ */
PAGE_TYPE_OPS(Offline, offline) PAGE_TYPE_OPS(Offline, offline)
......
...@@ -17,6 +17,16 @@ struct vdpa_callback { ...@@ -17,6 +17,16 @@ struct vdpa_callback {
void *private; void *private;
}; };
/**
* vDPA notification area
* @addr: base address of the notification area
* @size: size of the notification area
*/
struct vdpa_notification_area {
resource_size_t addr;
resource_size_t size;
};
/** /**
* vDPA device - representation of a vDPA device * vDPA device - representation of a vDPA device
* @dev: underlying device * @dev: underlying device
...@@ -73,6 +83,10 @@ struct vdpa_device { ...@@ -73,6 +83,10 @@ struct vdpa_device {
* @vdev: vdpa device * @vdev: vdpa device
* @idx: virtqueue index * @idx: virtqueue index
* Returns virtqueue state (last_avail_idx) * Returns virtqueue state (last_avail_idx)
* @get_vq_notification: Get the notification area for a virtqueue
* @vdev: vdpa device
* @idx: virtqueue index
* Returns the notifcation area
* @get_vq_align: Get the virtqueue align requirement * @get_vq_align: Get the virtqueue align requirement
* for the device * for the device
* @vdev: vdpa device * @vdev: vdpa device
...@@ -162,6 +176,8 @@ struct vdpa_config_ops { ...@@ -162,6 +176,8 @@ struct vdpa_config_ops {
bool (*get_vq_ready)(struct vdpa_device *vdev, u16 idx); bool (*get_vq_ready)(struct vdpa_device *vdev, u16 idx);
int (*set_vq_state)(struct vdpa_device *vdev, u16 idx, u64 state); int (*set_vq_state)(struct vdpa_device *vdev, u16 idx, u64 state);
u64 (*get_vq_state)(struct vdpa_device *vdev, u16 idx); u64 (*get_vq_state)(struct vdpa_device *vdev, u16 idx);
struct vdpa_notification_area
(*get_vq_notification)(struct vdpa_device *vdev, u16 idx);
/* Device ops */ /* Device ops */
u32 (*get_vq_align)(struct vdpa_device *vdev); u32 (*get_vq_align)(struct vdpa_device *vdev);
......
...@@ -105,9 +105,9 @@ struct vringh_kiov { ...@@ -105,9 +105,9 @@ struct vringh_kiov {
/* Helpers for userspace vrings. */ /* Helpers for userspace vrings. */
int vringh_init_user(struct vringh *vrh, u64 features, int vringh_init_user(struct vringh *vrh, u64 features,
unsigned int num, bool weak_barriers, unsigned int num, bool weak_barriers,
struct vring_desc __user *desc, vring_desc_t __user *desc,
struct vring_avail __user *avail, vring_avail_t __user *avail,
struct vring_used __user *used); vring_used_t __user *used);
static inline void vringh_iov_init(struct vringh_iov *iov, static inline void vringh_iov_init(struct vringh_iov *iov,
struct iovec *iovec, unsigned num) struct iovec *iovec, unsigned num)
......
...@@ -15,6 +15,8 @@ ...@@ -15,6 +15,8 @@
#include <linux/types.h> #include <linux/types.h>
#include <linux/ioctl.h> #include <linux/ioctl.h>
#define VHOST_FILE_UNBIND -1
/* ioctls */ /* ioctls */
#define VHOST_VIRTIO 0xAF #define VHOST_VIRTIO 0xAF
...@@ -140,4 +142,6 @@ ...@@ -140,4 +142,6 @@
/* Get the max ring size. */ /* Get the max ring size. */
#define VHOST_VDPA_GET_VRING_NUM _IOR(VHOST_VIRTIO, 0x76, __u16) #define VHOST_VDPA_GET_VRING_NUM _IOR(VHOST_VIRTIO, 0x76, __u16)
/* Set event fd for config interrupt*/
#define VHOST_VDPA_SET_CONFIG_CALL _IOW(VHOST_VIRTIO, 0x77, int)
#endif #endif
...@@ -44,6 +44,7 @@ ...@@ -44,6 +44,7 @@
#define VIRTIO_ID_VSOCK 19 /* virtio vsock transport */ #define VIRTIO_ID_VSOCK 19 /* virtio vsock transport */
#define VIRTIO_ID_CRYPTO 20 /* virtio crypto */ #define VIRTIO_ID_CRYPTO 20 /* virtio crypto */
#define VIRTIO_ID_IOMMU 23 /* virtio IOMMU */ #define VIRTIO_ID_IOMMU 23 /* virtio IOMMU */
#define VIRTIO_ID_MEM 24 /* virtio mem */
#define VIRTIO_ID_FS 26 /* virtio filesystem */ #define VIRTIO_ID_FS 26 /* virtio filesystem */
#define VIRTIO_ID_PMEM 27 /* virtio pmem */ #define VIRTIO_ID_PMEM 27 /* virtio pmem */
#define VIRTIO_ID_MAC80211_HWSIM 29 /* virtio mac80211-hwsim */ #define VIRTIO_ID_MAC80211_HWSIM 29 /* virtio mac80211-hwsim */
......
/* SPDX-License-Identifier: BSD-3-Clause */
/*
* Virtio Mem Device
*
* Copyright Red Hat, Inc. 2020
*
* Authors:
* David Hildenbrand <david@redhat.com>
*
* This header is BSD licensed so anyone can use the definitions
* to implement compatible drivers/servers:
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of IBM nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IBM OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _LINUX_VIRTIO_MEM_H
#define _LINUX_VIRTIO_MEM_H
#include <linux/types.h>
#include <linux/virtio_types.h>
#include <linux/virtio_ids.h>
#include <linux/virtio_config.h>
/*
* Each virtio-mem device manages a dedicated region in physical address
* space. Each device can belong to a single NUMA node, multiple devices
* for a single NUMA node are possible. A virtio-mem device is like a
* "resizable DIMM" consisting of small memory blocks that can be plugged
* or unplugged. The device driver is responsible for (un)plugging memory
* blocks on demand.
*
* Virtio-mem devices can only operate on their assigned memory region in
* order to (un)plug memory. A device cannot (un)plug memory belonging to
* other devices.
*
* The "region_size" corresponds to the maximum amount of memory that can
* be provided by a device. The "size" corresponds to the amount of memory
* that is currently plugged. "requested_size" corresponds to a request
* from the device to the device driver to (un)plug blocks. The
* device driver should try to (un)plug blocks in order to reach the
* "requested_size". It is impossible to plug more memory than requested.
*
* The "usable_region_size" represents the memory region that can actually
* be used to (un)plug memory. It is always at least as big as the
* "requested_size" and will grow dynamically. It will only shrink when
* explicitly triggered (VIRTIO_MEM_REQ_UNPLUG).
*
* There are no guarantees what will happen if unplugged memory is
* read/written. Such memory should, in general, not be touched. E.g.,
* even writing might succeed, but the values will simply be discarded at
* random points in time.
*
* It can happen that the device cannot process a request, because it is
* busy. The device driver has to retry later.
*
* Usually, during system resets all memory will get unplugged, so the
* device driver can start with a clean state. However, in specific
* scenarios (if the device is busy) it can happen that the device still
* has memory plugged. The device driver can request to unplug all memory
* (VIRTIO_MEM_REQ_UNPLUG) - which might take a while to succeed if the
* device is busy.
*/
/* --- virtio-mem: feature bits --- */
/* node_id is an ACPI PXM and is valid */
#define VIRTIO_MEM_F_ACPI_PXM 0
/* --- virtio-mem: guest -> host requests --- */
/* request to plug memory blocks */
#define VIRTIO_MEM_REQ_PLUG 0
/* request to unplug memory blocks */
#define VIRTIO_MEM_REQ_UNPLUG 1
/* request to unplug all blocks and shrink the usable size */
#define VIRTIO_MEM_REQ_UNPLUG_ALL 2
/* request information about the plugged state of memory blocks */
#define VIRTIO_MEM_REQ_STATE 3
struct virtio_mem_req_plug {
__virtio64 addr;
__virtio16 nb_blocks;
__virtio16 padding[3];
};
struct virtio_mem_req_unplug {
__virtio64 addr;
__virtio16 nb_blocks;
__virtio16 padding[3];
};
struct virtio_mem_req_state {
__virtio64 addr;
__virtio16 nb_blocks;
__virtio16 padding[3];
};
struct virtio_mem_req {
__virtio16 type;
__virtio16 padding[3];
union {
struct virtio_mem_req_plug plug;
struct virtio_mem_req_unplug unplug;
struct virtio_mem_req_state state;
} u;
};
/* --- virtio-mem: host -> guest response --- */
/*
* Request processed successfully, applicable for
* - VIRTIO_MEM_REQ_PLUG
* - VIRTIO_MEM_REQ_UNPLUG
* - VIRTIO_MEM_REQ_UNPLUG_ALL
* - VIRTIO_MEM_REQ_STATE
*/
#define VIRTIO_MEM_RESP_ACK 0
/*
* Request denied - e.g. trying to plug more than requested, applicable for
* - VIRTIO_MEM_REQ_PLUG
*/
#define VIRTIO_MEM_RESP_NACK 1
/*
* Request cannot be processed right now, try again later, applicable for
* - VIRTIO_MEM_REQ_PLUG
* - VIRTIO_MEM_REQ_UNPLUG
* - VIRTIO_MEM_REQ_UNPLUG_ALL
*/
#define VIRTIO_MEM_RESP_BUSY 2
/*
* Error in request (e.g. addresses/alignment), applicable for
* - VIRTIO_MEM_REQ_PLUG
* - VIRTIO_MEM_REQ_UNPLUG
* - VIRTIO_MEM_REQ_STATE
*/
#define VIRTIO_MEM_RESP_ERROR 3
/* State of memory blocks is "plugged" */
#define VIRTIO_MEM_STATE_PLUGGED 0
/* State of memory blocks is "unplugged" */
#define VIRTIO_MEM_STATE_UNPLUGGED 1
/* State of memory blocks is "mixed" */
#define VIRTIO_MEM_STATE_MIXED 2
struct virtio_mem_resp_state {
__virtio16 state;
};
struct virtio_mem_resp {
__virtio16 type;
__virtio16 padding[3];
union {
struct virtio_mem_resp_state state;
} u;
};
/* --- virtio-mem: configuration --- */
struct virtio_mem_config {
/* Block size and alignment. Cannot change. */
__u64 block_size;
/* Valid with VIRTIO_MEM_F_ACPI_PXM. Cannot change. */
__u16 node_id;
__u8 padding[6];
/* Start address of the memory region. Cannot change. */
__u64 addr;
/* Region size (maximum). Cannot change. */
__u64 region_size;
/*
* Currently usable region size. Can grow up to region_size. Can
* shrink due to VIRTIO_MEM_REQ_UNPLUG_ALL (in which case no config
* update will be sent).
*/
__u64 usable_region_size;
/*
* Currently used size. Changes due to plug/unplug requests, but no
* config updates will be sent.
*/
__u64 plugged_size;
/* Requested size. New plug requests cannot exceed it. Can change. */
__u64 requested_size;
};
#endif /* _LINUX_VIRTIO_MEM_H */
...@@ -86,6 +86,13 @@ ...@@ -86,6 +86,13 @@
* at the end of the used ring. Guest should ignore the used->flags field. */ * at the end of the used ring. Guest should ignore the used->flags field. */
#define VIRTIO_RING_F_EVENT_IDX 29 #define VIRTIO_RING_F_EVENT_IDX 29
/* Alignment requirements for vring elements.
* When using pre-virtio 1.0 layout, these fall out naturally.
*/
#define VRING_AVAIL_ALIGN_SIZE 2
#define VRING_USED_ALIGN_SIZE 4
#define VRING_DESC_ALIGN_SIZE 16
/* Virtio ring descriptors: 16 bytes. These can chain together via "next". */ /* Virtio ring descriptors: 16 bytes. These can chain together via "next". */
struct vring_desc { struct vring_desc {
/* Address (guest-physical). */ /* Address (guest-physical). */
...@@ -112,28 +119,47 @@ struct vring_used_elem { ...@@ -112,28 +119,47 @@ struct vring_used_elem {
__virtio32 len; __virtio32 len;
}; };
typedef struct vring_used_elem __attribute__((aligned(VRING_USED_ALIGN_SIZE)))
vring_used_elem_t;
struct vring_used { struct vring_used {
__virtio16 flags; __virtio16 flags;
__virtio16 idx; __virtio16 idx;
struct vring_used_elem ring[]; vring_used_elem_t ring[];
}; };
/*
* The ring element addresses are passed between components with different
* alignments assumptions. Thus, we might need to decrease the compiler-selected
* alignment, and so must use a typedef to make sure the aligned attribute
* actually takes hold:
*
* https://gcc.gnu.org/onlinedocs//gcc/Common-Type-Attributes.html#Common-Type-Attributes
*
* When used on a struct, or struct member, the aligned attribute can only
* increase the alignment; in order to decrease it, the packed attribute must
* be specified as well. When used as part of a typedef, the aligned attribute
* can both increase and decrease alignment, and specifying the packed
* attribute generates a warning.
*/
typedef struct vring_desc __attribute__((aligned(VRING_DESC_ALIGN_SIZE)))
vring_desc_t;
typedef struct vring_avail __attribute__((aligned(VRING_AVAIL_ALIGN_SIZE)))
vring_avail_t;
typedef struct vring_used __attribute__((aligned(VRING_USED_ALIGN_SIZE)))
vring_used_t;
struct vring { struct vring {
unsigned int num; unsigned int num;
struct vring_desc *desc; vring_desc_t *desc;
struct vring_avail *avail; vring_avail_t *avail;
struct vring_used *used; vring_used_t *used;
}; };
/* Alignment requirements for vring elements. #ifndef VIRTIO_RING_NO_LEGACY
* When using pre-virtio 1.0 layout, these fall out naturally.
*/
#define VRING_AVAIL_ALIGN_SIZE 2
#define VRING_USED_ALIGN_SIZE 4
#define VRING_DESC_ALIGN_SIZE 16
/* The standard layout for the ring is a continuous chunk of memory which looks /* The standard layout for the ring is a continuous chunk of memory which looks
* like this. We assume num is a power of 2. * like this. We assume num is a power of 2.
...@@ -181,6 +207,8 @@ static inline unsigned vring_size(unsigned int num, unsigned long align) ...@@ -181,6 +207,8 @@ static inline unsigned vring_size(unsigned int num, unsigned long align)
+ sizeof(__virtio16) * 3 + sizeof(struct vring_used_elem) * num; + sizeof(__virtio16) * 3 + sizeof(struct vring_used_elem) * num;
} }
#endif /* VIRTIO_RING_NO_LEGACY */
/* The following is used with USED_EVENT_IDX and AVAIL_EVENT_IDX */ /* The following is used with USED_EVENT_IDX and AVAIL_EVENT_IDX */
/* Assuming a given event_idx value from the other side, if /* Assuming a given event_idx value from the other side, if
* we have just incremented index from old to new_idx, * we have just incremented index from old to new_idx,
......
...@@ -1201,11 +1201,17 @@ struct zone *test_pages_in_a_zone(unsigned long start_pfn, ...@@ -1201,11 +1201,17 @@ struct zone *test_pages_in_a_zone(unsigned long start_pfn,
/* /*
* Scan pfn range [start,end) to find movable/migratable pages (LRU pages, * Scan pfn range [start,end) to find movable/migratable pages (LRU pages,
* non-lru movable pages and hugepages). We scan pfn because it's much * non-lru movable pages and hugepages). Will skip over most unmovable
* easier than scanning over linked list. This function returns the pfn * pages (esp., pages that can be skipped when offlining), but bail out on
* of the first found movable page if it's found, otherwise 0. * definitely unmovable pages.
*
* Returns:
* 0 in case a movable page is found and movable_pfn was updated.
* -ENOENT in case no movable page was found.
* -EBUSY in case a definitely unmovable page was found.
*/ */
static unsigned long scan_movable_pages(unsigned long start, unsigned long end) static int scan_movable_pages(unsigned long start, unsigned long end,
unsigned long *movable_pfn)
{ {
unsigned long pfn; unsigned long pfn;
...@@ -1217,18 +1223,30 @@ static unsigned long scan_movable_pages(unsigned long start, unsigned long end) ...@@ -1217,18 +1223,30 @@ static unsigned long scan_movable_pages(unsigned long start, unsigned long end)
continue; continue;
page = pfn_to_page(pfn); page = pfn_to_page(pfn);
if (PageLRU(page)) if (PageLRU(page))
return pfn; goto found;
if (__PageMovable(page)) if (__PageMovable(page))
return pfn; goto found;
/*
* PageOffline() pages that are not marked __PageMovable() and
* have a reference count > 0 (after MEM_GOING_OFFLINE) are
* definitely unmovable. If their reference count would be 0,
* they could at least be skipped when offlining memory.
*/
if (PageOffline(page) && page_count(page))
return -EBUSY;
if (!PageHuge(page)) if (!PageHuge(page))
continue; continue;
head = compound_head(page); head = compound_head(page);
if (page_huge_active(head)) if (page_huge_active(head))
return pfn; goto found;
skip = compound_nr(head) - (page - head); skip = compound_nr(head) - (page - head);
pfn += skip - 1; pfn += skip - 1;
} }
return -ENOENT;
found:
*movable_pfn = pfn;
return 0; return 0;
} }
...@@ -1491,7 +1509,8 @@ static int __ref __offline_pages(unsigned long start_pfn, ...@@ -1491,7 +1509,8 @@ static int __ref __offline_pages(unsigned long start_pfn,
} }
do { do {
for (pfn = start_pfn; pfn;) { pfn = start_pfn;
do {
if (signal_pending(current)) { if (signal_pending(current)) {
ret = -EINTR; ret = -EINTR;
reason = "signal backoff"; reason = "signal backoff";
...@@ -1501,14 +1520,19 @@ static int __ref __offline_pages(unsigned long start_pfn, ...@@ -1501,14 +1520,19 @@ static int __ref __offline_pages(unsigned long start_pfn,
cond_resched(); cond_resched();
lru_add_drain_all(); lru_add_drain_all();
pfn = scan_movable_pages(pfn, end_pfn); ret = scan_movable_pages(pfn, end_pfn, &pfn);
if (pfn) { if (!ret) {
/* /*
* TODO: fatal migration failures should bail * TODO: fatal migration failures should bail
* out * out
*/ */
do_migrate_range(pfn, end_pfn); do_migrate_range(pfn, end_pfn);
} }
} while (!ret);
if (ret != -ENOENT) {
reason = "unmovable page";
goto failed_removal_isolated;
} }
/* /*
...@@ -1774,4 +1798,41 @@ int remove_memory(int nid, u64 start, u64 size) ...@@ -1774,4 +1798,41 @@ int remove_memory(int nid, u64 start, u64 size)
return rc; return rc;
} }
EXPORT_SYMBOL_GPL(remove_memory); EXPORT_SYMBOL_GPL(remove_memory);
/*
* Try to offline and remove a memory block. Might take a long time to
* finish in case memory is still in use. Primarily useful for memory devices
* that logically unplugged all memory (so it's no longer in use) and want to
* offline + remove the memory block.
*/
int offline_and_remove_memory(int nid, u64 start, u64 size)
{
struct memory_block *mem;
int rc = -EINVAL;
if (!IS_ALIGNED(start, memory_block_size_bytes()) ||
size != memory_block_size_bytes())
return rc;
lock_device_hotplug();
mem = find_memory_block(__pfn_to_section(PFN_DOWN(start)));
if (mem)
rc = device_offline(&mem->dev);
/* Ignore if the device is already offline. */
if (rc > 0)
rc = 0;
/*
* In case we succeeded to offline the memory block, remove it.
* This cannot fail as it cannot get onlined in the meantime.
*/
if (!rc) {
rc = try_remove_memory(nid, start, size);
WARN_ON_ONCE(rc);
}
unlock_device_hotplug();
return rc;
}
EXPORT_SYMBOL_GPL(offline_and_remove_memory);
#endif /* CONFIG_MEMORY_HOTREMOVE */ #endif /* CONFIG_MEMORY_HOTREMOVE */
...@@ -8285,6 +8285,19 @@ struct page *has_unmovable_pages(struct zone *zone, struct page *page, ...@@ -8285,6 +8285,19 @@ struct page *has_unmovable_pages(struct zone *zone, struct page *page,
if ((flags & MEMORY_OFFLINE) && PageHWPoison(page)) if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))
continue; continue;
/*
* We treat all PageOffline() pages as movable when offlining
* to give drivers a chance to decrement their reference count
* in MEM_GOING_OFFLINE in order to indicate that these pages
* can be offlined as there are no direct references anymore.
* For actually unmovable PageOffline() where the driver does
* not support this, we will fail later when trying to actually
* move these pages that still have a reference count > 0.
* (false negatives in this function only)
*/
if ((flags & MEMORY_OFFLINE) && PageOffline(page))
continue;
if (__PageMovable(page) || PageLRU(page)) if (__PageMovable(page) || PageLRU(page))
continue; continue;
...@@ -8516,6 +8529,7 @@ int alloc_contig_range(unsigned long start, unsigned long end, ...@@ -8516,6 +8529,7 @@ int alloc_contig_range(unsigned long start, unsigned long end,
pfn_max_align_up(end), migratetype); pfn_max_align_up(end), migratetype);
return ret; return ret;
} }
EXPORT_SYMBOL(alloc_contig_range);
static int __alloc_contig_pages(unsigned long start_pfn, static int __alloc_contig_pages(unsigned long start_pfn,
unsigned long nr_pages, gfp_t gfp_mask) unsigned long nr_pages, gfp_t gfp_mask)
...@@ -8631,6 +8645,7 @@ void free_contig_range(unsigned long pfn, unsigned int nr_pages) ...@@ -8631,6 +8645,7 @@ void free_contig_range(unsigned long pfn, unsigned int nr_pages)
} }
WARN(count != 0, "%d pages are still in use!\n", count); WARN(count != 0, "%d pages are still in use!\n", count);
} }
EXPORT_SYMBOL(free_contig_range);
/* /*
* The zone indicated has a new number of managed_pages; batch sizes and percpu * The zone indicated has a new number of managed_pages; batch sizes and percpu
...@@ -8703,6 +8718,17 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) ...@@ -8703,6 +8718,17 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
offlined_pages++; offlined_pages++;
continue; continue;
} }
/*
* At this point all remaining PageOffline() pages have a
* reference count of 0 and can simply be skipped.
*/
if (PageOffline(page)) {
BUG_ON(page_count(page));
BUG_ON(PageBuddy(page));
pfn++;
offlined_pages++;
continue;
}
BUG_ON(page_count(page)); BUG_ON(page_count(page));
BUG_ON(!PageBuddy(page)); BUG_ON(!PageBuddy(page));
......
...@@ -151,6 +151,7 @@ __first_valid_page(unsigned long pfn, unsigned long nr_pages) ...@@ -151,6 +151,7 @@ __first_valid_page(unsigned long pfn, unsigned long nr_pages)
* a bit mask) * a bit mask)
* MEMORY_OFFLINE - isolate to offline (!allocate) memory * MEMORY_OFFLINE - isolate to offline (!allocate) memory
* e.g., skip over PageHWPoison() pages * e.g., skip over PageHWPoison() pages
* and PageOffline() pages.
* REPORT_FAILURE - report details about the failure to * REPORT_FAILURE - report details about the failure to
* isolate the range * isolate the range
* *
...@@ -259,6 +260,14 @@ __test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn, ...@@ -259,6 +260,14 @@ __test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn,
else if ((flags & MEMORY_OFFLINE) && PageHWPoison(page)) else if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))
/* A HWPoisoned page cannot be also PageBuddy */ /* A HWPoisoned page cannot be also PageBuddy */
pfn++; pfn++;
else if ((flags & MEMORY_OFFLINE) && PageOffline(page) &&
!page_count(page))
/*
* The responsible driver agreed to skip PageOffline()
* pages when offlining memory by dropping its
* reference in MEM_GOING_OFFLINE.
*/
pfn++;
else else
break; break;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment