From ea76b33ca7a8c2fd39f50b6d1bb6702ab0a4fc87 Mon Sep 17 00:00:00 2001 From: fangyi Date: Sat, 22 Jun 2024 07:02:48 +0000 Subject: [PATCH 1/6] vdpa: fix vdpa device migrate rollback wrong when suspend device failed. 1. set vdpa->suspended before call vhost_dev_suspend to make sure vdpa device will resume when suspend failed. 2. using state == RUN_STATE_FINISH_MIGRATE instead of ms->state == MIGRATION_STATUS_ACTIVE to judge vm in migration. As migrate_fd_cancel will change ms->state, which will result in some vdpa devices not being suspended. Signed-off-by: fangyi --- hw/virtio/vdpa-dev-mig.c | 81 ++++------------------------------------ 1 file changed, 7 insertions(+), 74 deletions(-) diff --git a/hw/virtio/vdpa-dev-mig.c b/hw/virtio/vdpa-dev-mig.c index 887c96a201b..7de996c8355 100644 --- a/hw/virtio/vdpa-dev-mig.c +++ b/hw/virtio/vdpa-dev-mig.c @@ -130,100 +130,33 @@ free: static int vhost_vdpa_device_suspend(VhostVdpaDevice *vdpa) { VirtIODevice *vdev = VIRTIO_DEVICE(vdpa); - BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); - VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); - int ret; if (!vdpa->started || vdpa->suspended) { return 0; } - if (!k->set_guest_notifiers) { - return -EFAULT; - } - - vdpa->started = false; vdpa->suspended = true; - ret = vhost_dev_suspend(&vdpa->dev, vdev, false); - if (ret) { - goto suspend_fail; - } - - ret = k->set_guest_notifiers(qbus->parent, vdpa->dev.nvqs, false); - if (ret < 0) { - error_report("vhost guest notifier cleanup failed: %d\n", ret); - goto set_guest_notifiers_fail; - } - - vhost_dev_disable_notifiers(&vdpa->dev, vdev); - return ret; - -set_guest_notifiers_fail: - ret = k->set_guest_notifiers(qbus->parent, vdpa->dev.nvqs, true); - if (ret) { - error_report("vhost guest notifier restore failed: %d\n", ret); - } - -suspend_fail: - vdpa->suspended = false; - vdpa->started = true; - return ret; + return vhost_dev_suspend(&vdpa->dev, vdev, false); } static int vhost_vdpa_device_resume(VhostVdpaDevice *vdpa) { VirtIODevice *vdev = VIRTIO_DEVICE(vdpa); - BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); - VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); - int i, ret; + MigrationIncomingState *mis = migration_incoming_get_current(); + int ret; - if (vdpa->started || !vdpa->suspended) { + if (!vdpa->started || + (!vdpa->suspended && mis->state != RUN_STATE_RESTORE_VM)) { return 0; } - if (!k->set_guest_notifiers) { - error_report("binding does not support guest notifiers\n"); - return -ENOSYS; - } - - ret = vhost_dev_enable_notifiers(&vdpa->dev, vdev); + ret = vhost_dev_resume(&vdpa->dev, vdev, false); if (ret < 0) { - error_report("Error enabling host notifiers: %d\n", ret); return ret; } - ret = k->set_guest_notifiers(qbus->parent, vdpa->dev.nvqs, true); - if (ret < 0) { - error_report("Error binding guest notifier: %d\n", ret); - goto err_host_notifiers; - } - - vdpa->dev.acked_features = vdev->guest_features; - - ret = vhost_dev_resume(&vdpa->dev, vdev, false); - if (ret < 0) { - error_report("Error starting vhost: %d\n", ret); - goto err_guest_notifiers; - } - vdpa->started = true; vdpa->suspended = false; - - /* - * guest_notifier_mask/pending not used yet, so just unmask - * everything here. virtio-pci will do the right thing by - * enabling/disabling irqfd. - */ - for (i = 0; i < vdpa->dev.nvqs; i++) { - vhost_virtqueue_mask(&vdpa->dev, vdev, i, false); - } - - return ret; - -err_guest_notifiers: - k->set_guest_notifiers(qbus->parent, vdpa->dev.nvqs, false); -err_host_notifiers: - vhost_dev_disable_notifiers(&vdpa->dev, vdev); return ret; } @@ -248,7 +181,7 @@ static void vdpa_dev_vmstate_change(void *opaque, bool running, RunState state) MigrationIncomingState *mis = migration_incoming_get_current(); if (!running) { - if (ms->state == MIGRATION_STATUS_ACTIVE || state == RUN_STATE_PAUSED) { + if (state == RUN_STATE_FINISH_MIGRATE || state == RUN_STATE_PAUSED) { ret = vhost_vdpa_device_suspend(vdpa); if (ret) { error_report("suspend vdpa device failed: %d\n", ret); -- Gitee From 8c65e8d7c923ade6f3c7fbef43000562d4733629 Mon Sep 17 00:00:00 2001 From: fangyi Date: Sat, 7 Sep 2024 07:11:07 +0000 Subject: [PATCH 2/6] vdpa: support resizing virtio-blk capacity online for kernel vdpa Signed-off-by: fangyi --- hw/virtio/vdpa-dev.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/hw/virtio/vdpa-dev.c b/hw/virtio/vdpa-dev.c index 91e71847b01..bf4b3ec3fd5 100644 --- a/hw/virtio/vdpa-dev.c +++ b/hw/virtio/vdpa-dev.c @@ -31,6 +31,7 @@ #include "hw/virtio/vdpa-dev-mig.h" #include "migration/migration.h" #include "exec/address-spaces.h" +#include "standard-headers/linux/virtio_ids.h" static void vhost_vdpa_device_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq) @@ -201,7 +202,23 @@ static void vhost_vdpa_device_get_config(VirtIODevice *vdev, uint8_t *config) { VhostVdpaDevice *s = VHOST_VDPA_DEVICE(vdev); + uint8_t *new_config; + int ret; + + if (s->vdev_id != VIRTIO_ID_BLOCK) { + goto out; + } + new_config = g_malloc0(s->config_size); + ret = vhost_dev_get_config(&s->dev, new_config, s->config_size, NULL); + if (ret < 0) { + error_report("vhost-vdpa-device: get config failed(%d)\n", ret); + goto free; + } + memcpy(s->config, new_config, s->config_size); +free: + g_free(new_config); +out: memcpy(config, s->config, s->config_size); } -- Gitee From 05ee3017d156005e3d8d8fb19514d593858abd44 Mon Sep 17 00:00:00 2001 From: fangyi Date: Tue, 29 Oct 2024 19:51:41 +0800 Subject: [PATCH 3/6] Revert "vdpa: add vhost_vdpa_suspend" Use a new scheme instead for kernel vdpa, So revert it. This reverts commit 0bb302a9960a186fc488068d268dc373e6b70876. --- hw/virtio/trace-events | 1 - hw/virtio/vhost-vdpa.c | 26 -------------------------- 2 files changed, 27 deletions(-) diff --git a/hw/virtio/trace-events b/hw/virtio/trace-events index 637cac4edf0..de02bdc1d02 100644 --- a/hw/virtio/trace-events +++ b/hw/virtio/trace-events @@ -52,7 +52,6 @@ vhost_vdpa_set_vring_ready(void *dev, unsigned i, int r) "dev: %p, idx: %u, r: % vhost_vdpa_dump_config(void *dev, const char *line) "dev: %p %s" vhost_vdpa_set_config(void *dev, uint32_t offset, uint32_t size, uint32_t flags) "dev: %p offset: %"PRIu32" size: %"PRIu32" flags: 0x%"PRIx32 vhost_vdpa_get_config(void *dev, void *config, uint32_t config_len) "dev: %p config: %p config_len: %"PRIu32 -vhost_vdpa_suspend(void *dev) "dev: %p" vhost_vdpa_dev_start(void *dev, bool started) "dev: %p started: %d" vhost_vdpa_set_log_base(void *dev, uint64_t base, unsigned long long size, int refcnt, int fd, void *log) "dev: %p base: 0x%"PRIx64" size: %llu refcnt: %d fd: %d log: %p" vhost_vdpa_set_vring_addr(void *dev, unsigned int index, unsigned int flags, uint64_t desc_user_addr, uint64_t used_user_addr, uint64_t avail_user_addr, uint64_t log_guest_addr) "dev: %p index: %u flags: 0x%x desc_user_addr: 0x%"PRIx64" used_user_addr: 0x%"PRIx64" avail_user_addr: 0x%"PRIx64" log_guest_addr: 0x%"PRIx64 diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c index d49826845fd..130afb06dc5 100644 --- a/hw/virtio/vhost-vdpa.c +++ b/hw/virtio/vhost-vdpa.c @@ -865,13 +865,11 @@ static int vhost_vdpa_get_device_id(struct vhost_dev *dev, static int vhost_vdpa_reset_device(struct vhost_dev *dev) { - struct vhost_vdpa *v = dev->opaque; int ret; uint8_t status = 0; ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status); trace_vhost_vdpa_reset_device(dev); - v->suspended = false; return ret; } @@ -1274,29 +1272,6 @@ static void vhost_vdpa_svqs_stop(struct vhost_dev *dev) } } -static void vhost_vdpa_suspend(struct vhost_dev *dev) -{ - struct vhost_vdpa *v = dev->opaque; - int r; - - if (!vhost_vdpa_first_dev(dev)) { - return; - } - - if (dev->backend_cap & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) { - trace_vhost_vdpa_suspend(dev); - r = ioctl(v->device_fd, VHOST_VDPA_SUSPEND); - if (unlikely(r)) { - error_report("Cannot suspend: %s(%d)", g_strerror(errno), errno); - } else { - v->suspended = true; - return; - } - } - - vhost_vdpa_reset_device(dev); -} - static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started) { struct vhost_vdpa *v = dev->opaque; @@ -1310,7 +1285,6 @@ static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started) return -1; } } else { - vhost_vdpa_suspend(dev); vhost_vdpa_svqs_stop(dev); vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); } -- Gitee From e1f733fcbc4eb39333ad9527865c1590d74092ed Mon Sep 17 00:00:00 2001 From: fangyi Date: Tue, 29 Oct 2024 19:53:27 +0800 Subject: [PATCH 4/6] Revert "vdpa: add vhost_vdpa->suspended parameter" Use a new scheme instead for kernel vdpa, So revert it. This reverts commit b6662cb7e5376659c7abb56efe27dcf3898d4fe6. --- hw/virtio/vhost-vdpa.c | 8 -------- include/hw/virtio/vhost-vdpa.h | 2 -- 2 files changed, 10 deletions(-) diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c index 130afb06dc5..bb3320946db 100644 --- a/hw/virtio/vhost-vdpa.c +++ b/hw/virtio/vhost-vdpa.c @@ -1406,14 +1406,6 @@ static int vhost_vdpa_get_vring_base(struct vhost_dev *dev, return 0; } - if (!v->suspended) { - /* - * Cannot trust in value returned by device, let vhost recover used - * idx from guest. - */ - return -1; - } - ret = vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring); trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num); return ret; diff --git a/include/hw/virtio/vhost-vdpa.h b/include/hw/virtio/vhost-vdpa.h index 5407d54fd79..ee255bc1bd7 100644 --- a/include/hw/virtio/vhost-vdpa.h +++ b/include/hw/virtio/vhost-vdpa.h @@ -42,8 +42,6 @@ typedef struct vhost_vdpa { bool shadow_vqs_enabled; /* Vdpa must send shadow addresses as IOTLB key for data queues, not GPA */ bool shadow_data; - /* Device suspended successfully */ - bool suspended; /* IOVA mapping used by the Shadow Virtqueue */ VhostIOVATree *iova_tree; GPtrArray *shadow_vqs; -- Gitee From 4a79b3c07dca4f1e21e4dbb1e59bf437b2a814fa Mon Sep 17 00:00:00 2001 From: fangyi Date: Tue, 29 Oct 2024 19:58:14 +0800 Subject: [PATCH 5/6] Revert "vdpa: block migration if SVQ does not admit a feature" Use a new scheme instead for kernel vdpa, So revert it. This reverts commit 57ac831865e370012496fb581a38d261cb72c5d0. --- hw/virtio/vhost-vdpa.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c index bb3320946db..69cf3b76e9e 100644 --- a/hw/virtio/vhost-vdpa.c +++ b/hw/virtio/vhost-vdpa.c @@ -596,21 +596,6 @@ static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp) return 0; } - /* - * If dev->shadow_vqs_enabled at initialization that means the device has - * been started with x-svq=on, so don't block migration - */ - if (dev->migration_blocker == NULL && !v->shadow_vqs_enabled) { - /* We don't have dev->features yet */ - uint64_t features; - ret = vhost_vdpa_get_dev_features(dev, &features); - if (unlikely(ret)) { - error_setg_errno(errp, -ret, "Could not get device features"); - return ret; - } - vhost_svq_valid_features(features, &dev->migration_blocker); - } - /* * Similar to VFIO, we end up pinning all guest memory and have to * disable discarding of RAM. -- Gitee From 1c62372d7c9e1f71ef9563e88b7491a7272b2a7d Mon Sep 17 00:00:00 2001 From: fangyi Date: Tue, 29 Oct 2024 20:02:10 +0800 Subject: [PATCH 6/6] vdpa: remove memory listener unregister in vhost_vdpa_reset_status Remove memory listener unregister in vhost_vdpa_reset_status as we move the memory listener registration of vdpa from the start stage to the realize stage before. --- hw/virtio/vhost-vdpa.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c index 69cf3b76e9e..dcf1ef2c152 100644 --- a/hw/virtio/vhost-vdpa.c +++ b/hw/virtio/vhost-vdpa.c @@ -1292,8 +1292,6 @@ static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started) static void vhost_vdpa_reset_status(struct vhost_dev *dev) { - struct vhost_vdpa *v = dev->opaque; - if (dev->vq_index + dev->nvqs != dev->vq_index_end) { return; } @@ -1301,7 +1299,6 @@ static void vhost_vdpa_reset_status(struct vhost_dev *dev) vhost_vdpa_reset_device(dev); vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER); - memory_listener_unregister(&v->listener); } static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base, -- Gitee