| 1 | /* |
| 2 | * Migration support for VFIO devices |
| 3 | * |
| 4 | * Copyright NVIDIA, Inc. 2020 |
| 5 | * |
| 6 | * This work is licensed under the terms of the GNU GPL, version 2. See |
| 7 | * the COPYING file in the top-level directory. |
| 8 | */ |
| 9 | |
| 10 | #include "qemu/osdep.h" |
| 11 | #include "qemu/main-loop.h" |
| 12 | #include "qemu/cutils.h" |
| 13 | #include "qemu/units.h" |
| 14 | #include "qemu/error-report.h" |
| 15 | #include <linux/vfio.h> |
| 16 | #include <sys/ioctl.h> |
| 17 | |
| 18 | #include "system/runstate.h" |
| 19 | #include "hw/core/boards.h" |
| 20 | #include "hw/vfio/vfio-device.h" |
| 21 | #include "hw/vfio/vfio-migration.h" |
| 22 | #include "migration/misc.h" |
| 23 | #include "migration/savevm.h" |
| 24 | #include "migration/vmstate.h" |
| 25 | #include "migration/qemu-file.h" |
| 26 | #include "migration/register.h" |
| 27 | #include "migration/blocker.h" |
| 28 | #include "migration-multifd.h" |
| 29 | #include "qapi/error.h" |
| 30 | #include "qapi/qapi-events-vfio.h" |
| 31 | #include "system/ramlist.h" |
| 32 | #include "pci.h" |
| 33 | #include "trace.h" |
| 34 | #include "hw/core/hw-error.h" |
| 35 | #include "vfio-migration-internal.h" |
| 36 | |
| 37 | /* |
| 38 | * This is an arbitrary size based on migration of mlx5 devices, where typically |
| 39 | * total device migration size is on the order of 100s of MB. Testing with |
| 40 | * larger values, e.g. 128MB and 1GB, did not show a performance improvement. |
| 41 | */ |
| 42 | #define VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE (1 * MiB) |
| 43 | |
| 44 | /* |
| 45 | * Migration size of VFIO devices can be as little as a few KBs or as big as |
| 46 | * many GBs. This value should be big enough to cover the worst case. |
| 47 | */ |
| 48 | #define VFIO_MIG_STOP_COPY_SIZE (100 * GiB) |
| 49 | |
| 50 | static unsigned long bytes_transferred; |
| 51 | |
| 52 | static const char *mig_state_to_str(enum vfio_device_mig_state state) |
| 53 | { |
| 54 | switch (state) { |
| 55 | case VFIO_DEVICE_STATE_ERROR: |
| 56 | return "ERROR"; |
| 57 | case VFIO_DEVICE_STATE_STOP: |
| 58 | return "STOP"; |
| 59 | case VFIO_DEVICE_STATE_RUNNING: |
| 60 | return "RUNNING"; |
| 61 | case VFIO_DEVICE_STATE_STOP_COPY: |
| 62 | return "STOP_COPY"; |
| 63 | case VFIO_DEVICE_STATE_RESUMING: |
| 64 | return "RESUMING"; |
| 65 | case VFIO_DEVICE_STATE_RUNNING_P2P: |
| 66 | return "RUNNING_P2P"; |
| 67 | case VFIO_DEVICE_STATE_PRE_COPY: |
| 68 | return "PRE_COPY"; |
| 69 | case VFIO_DEVICE_STATE_PRE_COPY_P2P: |
| 70 | return "PRE_COPY_P2P"; |
| 71 | default: |
| 72 | return "UNKNOWN STATE"; |
| 73 | } |
| 74 | } |
| 75 | |
| 76 | static QapiVfioMigrationState |
| 77 | mig_state_to_qapi_state(enum vfio_device_mig_state state, bool prepare) |
| 78 | { |
| 79 | switch (state) { |
| 80 | case VFIO_DEVICE_STATE_STOP: |
| 81 | return QAPI_VFIO_MIGRATION_STATE_STOP; |
| 82 | case VFIO_DEVICE_STATE_RUNNING: |
| 83 | return QAPI_VFIO_MIGRATION_STATE_RUNNING; |
| 84 | case VFIO_DEVICE_STATE_STOP_COPY: |
| 85 | return QAPI_VFIO_MIGRATION_STATE_STOP_COPY; |
| 86 | case VFIO_DEVICE_STATE_RESUMING: |
| 87 | return QAPI_VFIO_MIGRATION_STATE_RESUMING; |
| 88 | case VFIO_DEVICE_STATE_RUNNING_P2P: |
| 89 | return QAPI_VFIO_MIGRATION_STATE_RUNNING_P2P; |
| 90 | case VFIO_DEVICE_STATE_PRE_COPY: |
| 91 | return QAPI_VFIO_MIGRATION_STATE_PRE_COPY; |
| 92 | case VFIO_DEVICE_STATE_PRE_COPY_P2P: |
| 93 | return prepare ? QAPI_VFIO_MIGRATION_STATE_PRE_COPY_P2P_PREPARE : |
| 94 | QAPI_VFIO_MIGRATION_STATE_PRE_COPY_P2P; |
| 95 | default: |
| 96 | g_assert_not_reached(); |
| 97 | } |
| 98 | } |
| 99 | |
| 100 | static void vfio_migration_send_event(VFIODevice *vbasedev, |
| 101 | enum vfio_device_mig_state state, |
| 102 | bool prepare) |
| 103 | { |
| 104 | DeviceState *dev = vbasedev->dev; |
| 105 | g_autofree char *qom_path = NULL; |
| 106 | Object *obj; |
| 107 | |
| 108 | if (!vbasedev->migration_events) { |
| 109 | return; |
| 110 | } |
| 111 | |
| 112 | g_assert(vbasedev->ops->vfio_get_object); |
| 113 | obj = vbasedev->ops->vfio_get_object(vbasedev); |
| 114 | g_assert(obj); |
| 115 | qom_path = object_get_canonical_path(obj); |
| 116 | |
| 117 | qapi_event_send_vfio_migration(dev->id, qom_path, |
| 118 | mig_state_to_qapi_state(state, prepare)); |
| 119 | } |
| 120 | |
| 121 | static void vfio_migration_set_device_state(VFIODevice *vbasedev, |
| 122 | enum vfio_device_mig_state state) |
| 123 | { |
| 124 | VFIOMigration *migration = vbasedev->migration; |
| 125 | |
| 126 | trace_vfio_migration_set_device_state(vbasedev->name, |
| 127 | mig_state_to_str(state)); |
| 128 | |
| 129 | migration->device_state = state; |
| 130 | vfio_migration_send_event(vbasedev, state, false); |
| 131 | } |
| 132 | |
| 133 | int vfio_migration_set_state(VFIODevice *vbasedev, |
| 134 | enum vfio_device_mig_state new_state, |
| 135 | enum vfio_device_mig_state recover_state, |
| 136 | Error **errp) |
| 137 | { |
| 138 | VFIOMigration *migration = vbasedev->migration; |
| 139 | uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) + |
| 140 | sizeof(struct vfio_device_feature_mig_state), |
| 141 | sizeof(uint64_t))] = {}; |
| 142 | struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; |
| 143 | struct vfio_device_feature_mig_state *mig_state = |
| 144 | (struct vfio_device_feature_mig_state *)feature->data; |
| 145 | int ret; |
| 146 | g_autofree char *error_prefix = |
| 147 | g_strdup_printf("%s: Failed setting device state to %s.", |
| 148 | vbasedev->name, mig_state_to_str(new_state)); |
| 149 | |
| 150 | trace_vfio_migration_set_state(vbasedev->name, mig_state_to_str(new_state), |
| 151 | mig_state_to_str(recover_state)); |
| 152 | |
| 153 | if (new_state == migration->device_state) { |
| 154 | return 0; |
| 155 | } |
| 156 | |
| 157 | /* |
| 158 | * Send a prepare event before initiating the PRE_COPY_P2P transition to |
| 159 | * ensure timely event delivery regardless of how long the state transition |
| 160 | * takes. |
| 161 | */ |
| 162 | if (new_state == VFIO_DEVICE_STATE_PRE_COPY_P2P) { |
| 163 | vfio_migration_send_event(vbasedev, VFIO_DEVICE_STATE_PRE_COPY_P2P, |
| 164 | true); |
| 165 | } |
| 166 | |
| 167 | feature->argsz = sizeof(buf); |
| 168 | feature->flags = |
| 169 | VFIO_DEVICE_FEATURE_SET | VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE; |
| 170 | mig_state->device_state = new_state; |
| 171 | if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { |
| 172 | /* Try to set the device in some good state */ |
| 173 | ret = -errno; |
| 174 | |
| 175 | if (recover_state == VFIO_DEVICE_STATE_ERROR) { |
| 176 | error_setg_errno(errp, errno, |
| 177 | "%s Recover state is ERROR. Resetting device", |
| 178 | error_prefix); |
| 179 | |
| 180 | goto reset_device; |
| 181 | } |
| 182 | |
| 183 | error_setg_errno(errp, errno, |
| 184 | "%s Setting device in recover state %s", |
| 185 | error_prefix, mig_state_to_str(recover_state)); |
| 186 | |
| 187 | mig_state->device_state = recover_state; |
| 188 | if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { |
| 189 | ret = -errno; |
| 190 | /* |
| 191 | * If setting the device in recover state fails, report |
| 192 | * the error here and propagate the first error. |
| 193 | */ |
| 194 | error_report( |
| 195 | "%s: Failed setting device in recover state, err: %s. Resetting device", |
| 196 | vbasedev->name, strerror(errno)); |
| 197 | |
| 198 | goto reset_device; |
| 199 | } |
| 200 | |
| 201 | vfio_migration_set_device_state(vbasedev, recover_state); |
| 202 | |
| 203 | return ret; |
| 204 | } |
| 205 | |
| 206 | vfio_migration_set_device_state(vbasedev, new_state); |
| 207 | if (mig_state->data_fd != -1) { |
| 208 | if (migration->data_fd != -1) { |
| 209 | /* |
| 210 | * This can happen if the device is asynchronously reset and |
| 211 | * terminates a data transfer. |
| 212 | */ |
| 213 | error_setg(errp, "%s: data_fd out of sync", vbasedev->name); |
| 214 | close(mig_state->data_fd); |
| 215 | |
| 216 | return -EBADF; |
| 217 | } |
| 218 | |
| 219 | migration->data_fd = mig_state->data_fd; |
| 220 | } |
| 221 | |
| 222 | return 0; |
| 223 | |
| 224 | reset_device: |
| 225 | if (ioctl(vbasedev->fd, VFIO_DEVICE_RESET)) { |
| 226 | hw_error("%s: Failed resetting device, err: %s", vbasedev->name, |
| 227 | strerror(errno)); |
| 228 | } |
| 229 | |
| 230 | vfio_migration_set_device_state(vbasedev, VFIO_DEVICE_STATE_RUNNING); |
| 231 | |
| 232 | return ret; |
| 233 | } |
| 234 | |
| 235 | /* |
| 236 | * Some device state transitions require resetting the device if they fail. |
| 237 | * This function sets the device in new_state and resets the device if that |
| 238 | * fails. Reset is done by using ERROR as the recover state. |
| 239 | */ |
| 240 | static int |
| 241 | vfio_migration_set_state_or_reset(VFIODevice *vbasedev, |
| 242 | enum vfio_device_mig_state new_state, |
| 243 | Error **errp) |
| 244 | { |
| 245 | return vfio_migration_set_state(vbasedev, new_state, |
| 246 | VFIO_DEVICE_STATE_ERROR, errp); |
| 247 | } |
| 248 | |
| 249 | static int vfio_load_buffer(QEMUFile *f, VFIODevice *vbasedev, |
| 250 | uint64_t data_size) |
| 251 | { |
| 252 | VFIOMigration *migration = vbasedev->migration; |
| 253 | int ret; |
| 254 | |
| 255 | ret = qemu_file_get_to_fd(f, migration->data_fd, data_size); |
| 256 | trace_vfio_load_state_device_data(vbasedev->name, data_size, ret); |
| 257 | |
| 258 | return ret; |
| 259 | } |
| 260 | |
| 261 | int vfio_save_device_config_state(QEMUFile *f, void *opaque, Error **errp) |
| 262 | { |
| 263 | VFIODevice *vbasedev = opaque; |
| 264 | int ret; |
| 265 | |
| 266 | qemu_put_be64(f, VFIO_MIG_FLAG_DEV_CONFIG_STATE); |
| 267 | |
| 268 | if (vbasedev->ops && vbasedev->ops->vfio_save_config) { |
| 269 | ret = vbasedev->ops->vfio_save_config(vbasedev, f, errp); |
| 270 | if (ret) { |
| 271 | return ret; |
| 272 | } |
| 273 | } |
| 274 | |
| 275 | qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE); |
| 276 | |
| 277 | trace_vfio_save_device_config_state(vbasedev->name); |
| 278 | |
| 279 | ret = qemu_file_get_error(f); |
| 280 | if (ret < 0) { |
| 281 | error_setg_errno(errp, -ret, "Failed to save state"); |
| 282 | } |
| 283 | return ret; |
| 284 | } |
| 285 | |
| 286 | int vfio_load_device_config_state(QEMUFile *f, void *opaque) |
| 287 | { |
| 288 | VFIODevice *vbasedev = opaque; |
| 289 | uint64_t data; |
| 290 | |
| 291 | trace_vfio_load_device_config_state_start(vbasedev->name); |
| 292 | |
| 293 | if (vbasedev->ops && vbasedev->ops->vfio_load_config) { |
| 294 | int ret; |
| 295 | |
| 296 | ret = vbasedev->ops->vfio_load_config(vbasedev, f); |
| 297 | if (ret) { |
| 298 | error_report("%s: Failed to load device config space", |
| 299 | vbasedev->name); |
| 300 | return ret; |
| 301 | } |
| 302 | } |
| 303 | |
| 304 | data = qemu_get_be64(f); |
| 305 | if (data != VFIO_MIG_FLAG_END_OF_STATE) { |
| 306 | error_report("%s: Failed loading device config space, " |
| 307 | "end flag incorrect 0x%"PRIx64, vbasedev->name, data); |
| 308 | return -EINVAL; |
| 309 | } |
| 310 | |
| 311 | trace_vfio_load_device_config_state_end(vbasedev->name); |
| 312 | return qemu_file_get_error(f); |
| 313 | } |
| 314 | |
| 315 | static void vfio_migration_cleanup(VFIODevice *vbasedev) |
| 316 | { |
| 317 | VFIOMigration *migration = vbasedev->migration; |
| 318 | |
| 319 | close(migration->data_fd); |
| 320 | migration->data_fd = -1; |
| 321 | } |
| 322 | |
| 323 | static bool vfio_migration_check_overflow(VFIODevice *vbasedev, uint64_t size, |
| 324 | const char *name) |
| 325 | { |
| 326 | if (size > INT64_MAX) { |
| 327 | error_report("%s: Estimated %s size overflow: 0x%"PRIx64, |
| 328 | vbasedev->name, name, size); |
| 329 | return true; |
| 330 | } |
| 331 | |
| 332 | return false; |
| 333 | } |
| 334 | |
| 335 | static int vfio_query_stop_copy_size(VFIODevice *vbasedev) |
| 336 | { |
| 337 | uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) + |
| 338 | sizeof(struct vfio_device_feature_mig_data_size), |
| 339 | sizeof(uint64_t))] = {}; |
| 340 | struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; |
| 341 | struct vfio_device_feature_mig_data_size *mig_data_size = |
| 342 | (struct vfio_device_feature_mig_data_size *)feature->data; |
| 343 | VFIOMigration *migration = vbasedev->migration; |
| 344 | int ret = 0; |
| 345 | |
| 346 | feature->argsz = sizeof(buf); |
| 347 | feature->flags = |
| 348 | VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_MIG_DATA_SIZE; |
| 349 | |
| 350 | if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { |
| 351 | /* |
| 352 | * If getting pending migration size fails, VFIO_MIG_STOP_COPY_SIZE |
| 353 | * is reported so downtime limit won't be violated. |
| 354 | */ |
| 355 | migration->stopcopy_size = VFIO_MIG_STOP_COPY_SIZE; |
| 356 | ret = -errno; |
| 357 | warn_report_once("VFIO device %s ioctl(VFIO_DEVICE_FEATURE) on " |
| 358 | "VFIO_DEVICE_FEATURE_MIG_DATA_SIZE failed (%d)", |
| 359 | vbasedev->name, ret); |
| 360 | } else { |
| 361 | migration->stopcopy_size = mig_data_size->stop_copy_length; |
| 362 | if (vfio_migration_check_overflow(vbasedev, migration->stopcopy_size, |
| 363 | "stop copy size")) { |
| 364 | ret = -ERANGE; |
| 365 | } |
| 366 | } |
| 367 | |
| 368 | trace_vfio_query_stop_copy_size(vbasedev->name, |
| 369 | migration->stopcopy_size, ret); |
| 370 | |
| 371 | return ret; |
| 372 | } |
| 373 | |
| 374 | static int vfio_query_precopy_size(VFIOMigration *migration) |
| 375 | { |
| 376 | VFIODevice *vbasedev = migration->vbasedev; |
| 377 | struct vfio_precopy_info precopy = { |
| 378 | .argsz = sizeof(precopy), |
| 379 | }; |
| 380 | bool reinit = false; |
| 381 | int ret = 0; |
| 382 | |
| 383 | if (ioctl(migration->data_fd, VFIO_MIG_GET_PRECOPY_INFO, &precopy)) { |
| 384 | migration->precopy_init_size = 0; |
| 385 | migration->precopy_dirty_size = 0; |
| 386 | ret = -errno; |
| 387 | warn_report_once("VFIO device %s ioctl(VFIO_MIG_GET_PRECOPY_INFO) " |
| 388 | "failed (%d)", vbasedev->name, ret); |
| 389 | } else { |
| 390 | bool overflow; |
| 391 | |
| 392 | migration->precopy_init_size = precopy.initial_bytes; |
| 393 | migration->precopy_dirty_size = precopy.dirty_bytes; |
| 394 | /* |
| 395 | * struct vfio_precopy_info.flags is valid only if |
| 396 | * VFIO_DEVICE_FEATURE_MIG_PRECOPY_INFOv2 is used. |
| 397 | */ |
| 398 | if (migration->precopy_info_v2_used) { |
| 399 | reinit = precopy.flags & VFIO_PRECOPY_INFO_REINIT; |
| 400 | } |
| 401 | |
| 402 | overflow = vfio_migration_check_overflow(vbasedev, |
| 403 | migration->precopy_init_size, "precopy init size"); |
| 404 | overflow |= vfio_migration_check_overflow(vbasedev, |
| 405 | migration->precopy_dirty_size, "precopy dirty size"); |
| 406 | if (overflow) { |
| 407 | ret = -ERANGE; |
| 408 | } |
| 409 | } |
| 410 | |
| 411 | trace_vfio_query_precopy_size(vbasedev->name, migration->precopy_init_size, |
| 412 | migration->precopy_dirty_size, reinit, ret); |
| 413 | |
| 414 | /* |
| 415 | * If we got new initial_bytes after previous initial_bytes were |
| 416 | * transferred, request a new switchover ACK. Don't request if legacy |
| 417 | * switchover-ack is used. |
| 418 | */ |
| 419 | if (reinit && migration->initial_data_sent && |
| 420 | !migrate_switchover_ack_legacy()) { |
| 421 | migration->initial_data_sent = false; |
| 422 | migration->request_switchover_ack = true; |
| 423 | trace_vfio_query_precopy_size_request_switchover_ack(vbasedev->name); |
| 424 | } |
| 425 | |
| 426 | return ret; |
| 427 | } |
| 428 | |
| 429 | /* Returns the size of saved data on success and -errno on error */ |
| 430 | static ssize_t vfio_save_block(QEMUFile *f, VFIOMigration *migration) |
| 431 | { |
| 432 | ssize_t data_size; |
| 433 | |
| 434 | data_size = read(migration->data_fd, migration->data_buffer, |
| 435 | migration->data_buffer_size); |
| 436 | if (data_size < 0) { |
| 437 | /* |
| 438 | * Pre-copy emptied all the device state for now. For more information, |
| 439 | * please refer to the Linux kernel VFIO uAPI. |
| 440 | */ |
| 441 | if (errno == ENOMSG) { |
| 442 | if (!migration->event_precopy_empty_hit) { |
| 443 | trace_vfio_save_block_precopy_empty_hit(migration->vbasedev->name); |
| 444 | migration->event_precopy_empty_hit = true; |
| 445 | } |
| 446 | return 0; |
| 447 | } |
| 448 | |
| 449 | return -errno; |
| 450 | } |
| 451 | if (data_size == 0) { |
| 452 | return 0; |
| 453 | } |
| 454 | |
| 455 | /* Non-empty read: re-arm the trace event */ |
| 456 | migration->event_precopy_empty_hit = false; |
| 457 | |
| 458 | qemu_put_be64(f, VFIO_MIG_FLAG_DEV_DATA_STATE); |
| 459 | qemu_put_be64(f, data_size); |
| 460 | qemu_put_buffer(f, migration->data_buffer, data_size); |
| 461 | vfio_migration_add_bytes_transferred(data_size); |
| 462 | |
| 463 | trace_vfio_save_block(migration->vbasedev->name, data_size); |
| 464 | |
| 465 | return qemu_file_get_error(f) ?: data_size; |
| 466 | } |
| 467 | |
| 468 | static void vfio_update_estimated_pending_data(VFIOMigration *migration, |
| 469 | uint64_t data_size) |
| 470 | { |
| 471 | if (!data_size) { |
| 472 | /* |
| 473 | * Pre-copy emptied all the device state for now, update estimated sizes |
| 474 | * accordingly. |
| 475 | */ |
| 476 | migration->precopy_init_size = 0; |
| 477 | migration->precopy_dirty_size = 0; |
| 478 | |
| 479 | return; |
| 480 | } |
| 481 | |
| 482 | /* |
| 483 | * The total size remaining requires separate accounting. Do not trust |
| 484 | * the counter, so what we have read() may be more than what reported. |
| 485 | */ |
| 486 | if (migration->stopcopy_size > data_size) { |
| 487 | migration->stopcopy_size -= data_size; |
| 488 | } else { |
| 489 | migration->stopcopy_size = 0; |
| 490 | } |
| 491 | |
| 492 | if (migration->precopy_init_size) { |
| 493 | uint64_t init_size = MIN(migration->precopy_init_size, data_size); |
| 494 | |
| 495 | migration->precopy_init_size -= init_size; |
| 496 | data_size -= init_size; |
| 497 | } |
| 498 | |
| 499 | migration->precopy_dirty_size -= MIN(migration->precopy_dirty_size, |
| 500 | data_size); |
| 501 | } |
| 502 | |
| 503 | /* Returns true if the init data flag was sent, false otherwise */ |
| 504 | static bool vfio_send_init_data_flag(QEMUFile *f, VFIOMigration *migration) |
| 505 | { |
| 506 | VFIODevice *vbasedev = migration->vbasedev; |
| 507 | |
| 508 | if (!migrate_switchover_ack()) { |
| 509 | return false; |
| 510 | } |
| 511 | |
| 512 | if (migration->precopy_init_size || migration->initial_data_sent) { |
| 513 | return false; |
| 514 | } |
| 515 | |
| 516 | qemu_put_be64(f, VFIO_MIG_FLAG_DEV_INIT_DATA_SENT); |
| 517 | migration->initial_data_sent = true; |
| 518 | trace_vfio_send_init_data_flag(vbasedev->name); |
| 519 | |
| 520 | return true; |
| 521 | } |
| 522 | |
| 523 | static bool vfio_precopy_supported(VFIODevice *vbasedev) |
| 524 | { |
| 525 | VFIOMigration *migration = vbasedev->migration; |
| 526 | |
| 527 | return migration->mig_flags & VFIO_MIGRATION_PRE_COPY; |
| 528 | } |
| 529 | |
| 530 | static void vfio_request_switchover_ack_legacy(VFIODevice *vbasedev) |
| 531 | { |
| 532 | if (vfio_precopy_supported(vbasedev)) { |
| 533 | /* Precopy support implies switchover-ack is needed */ |
| 534 | migration_request_switchover_ack_legacy(vbasedev->name); |
| 535 | } |
| 536 | } |
| 537 | |
| 538 | /* ---------------------------------------------------------------------- */ |
| 539 | |
| 540 | static int vfio_save_prepare(void *opaque, Error **errp) |
| 541 | { |
| 542 | VFIODevice *vbasedev = opaque; |
| 543 | |
| 544 | /* |
| 545 | * Snapshot doesn't use postcopy nor background snapshot, so allow snapshot |
| 546 | * even if they are on. |
| 547 | */ |
| 548 | if (runstate_check(RUN_STATE_SAVE_VM)) { |
| 549 | return 0; |
| 550 | } |
| 551 | |
| 552 | if (migrate_postcopy_ram()) { |
| 553 | error_setg( |
| 554 | errp, "%s: VFIO migration is not supported with postcopy migration", |
| 555 | vbasedev->name); |
| 556 | return -EOPNOTSUPP; |
| 557 | } |
| 558 | |
| 559 | if (migrate_background_snapshot()) { |
| 560 | error_setg( |
| 561 | errp, |
| 562 | "%s: VFIO migration is not supported with background snapshot", |
| 563 | vbasedev->name); |
| 564 | return -EOPNOTSUPP; |
| 565 | } |
| 566 | |
| 567 | return 0; |
| 568 | } |
| 569 | |
| 570 | static int vfio_save_setup(QEMUFile *f, void *opaque, Error **errp) |
| 571 | { |
| 572 | VFIODevice *vbasedev = opaque; |
| 573 | VFIOMigration *migration = vbasedev->migration; |
| 574 | int ret; |
| 575 | |
| 576 | if (!vfio_multifd_setup(vbasedev, false, errp)) { |
| 577 | return -EINVAL; |
| 578 | } |
| 579 | |
| 580 | qemu_put_be64(f, VFIO_MIG_FLAG_DEV_SETUP_STATE); |
| 581 | |
| 582 | vfio_query_stop_copy_size(vbasedev); |
| 583 | migration->data_buffer_size = MIN(VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE, |
| 584 | migration->stopcopy_size); |
| 585 | migration->data_buffer = g_try_malloc0(migration->data_buffer_size); |
| 586 | if (!migration->data_buffer) { |
| 587 | error_setg(errp, "%s: Failed to allocate migration data buffer", |
| 588 | vbasedev->name); |
| 589 | return -ENOMEM; |
| 590 | } |
| 591 | |
| 592 | migration->event_save_iterate_started = false; |
| 593 | migration->event_precopy_empty_hit = false; |
| 594 | |
| 595 | if (vfio_precopy_supported(vbasedev)) { |
| 596 | switch (migration->device_state) { |
| 597 | case VFIO_DEVICE_STATE_RUNNING: |
| 598 | ret = vfio_migration_set_state(vbasedev, VFIO_DEVICE_STATE_PRE_COPY, |
| 599 | VFIO_DEVICE_STATE_RUNNING, errp); |
| 600 | if (ret) { |
| 601 | return ret; |
| 602 | } |
| 603 | |
| 604 | vfio_query_precopy_size(migration); |
| 605 | if (migrate_switchover_ack() && !migrate_switchover_ack_legacy()) { |
| 606 | migration->request_switchover_ack = true; |
| 607 | } |
| 608 | |
| 609 | break; |
| 610 | case VFIO_DEVICE_STATE_STOP: |
| 611 | /* vfio_save_complete_precopy() will go to STOP_COPY */ |
| 612 | break; |
| 613 | default: |
| 614 | error_setg(errp, "%s: Invalid device state %d", vbasedev->name, |
| 615 | migration->device_state); |
| 616 | return -EINVAL; |
| 617 | } |
| 618 | } |
| 619 | |
| 620 | trace_vfio_save_setup(vbasedev->name, migration->data_buffer_size); |
| 621 | |
| 622 | qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE); |
| 623 | |
| 624 | ret = qemu_file_get_error(f); |
| 625 | if (ret < 0) { |
| 626 | error_setg_errno(errp, -ret, "%s: save setup failed", vbasedev->name); |
| 627 | } |
| 628 | |
| 629 | return ret; |
| 630 | } |
| 631 | |
| 632 | static void vfio_save_cleanup(void *opaque) |
| 633 | { |
| 634 | VFIODevice *vbasedev = opaque; |
| 635 | VFIOMigration *migration = vbasedev->migration; |
| 636 | Error *local_err = NULL; |
| 637 | int ret; |
| 638 | |
| 639 | /* Currently a NOP, done for symmetry with load_cleanup() */ |
| 640 | vfio_multifd_cleanup(vbasedev); |
| 641 | |
| 642 | /* |
| 643 | * Changing device state from STOP_COPY to STOP can take time. Do it here, |
| 644 | * after migration has completed, so it won't increase downtime. |
| 645 | */ |
| 646 | if (migration->device_state == VFIO_DEVICE_STATE_STOP_COPY) { |
| 647 | ret = vfio_migration_set_state_or_reset(vbasedev, |
| 648 | VFIO_DEVICE_STATE_STOP, |
| 649 | &local_err); |
| 650 | if (ret) { |
| 651 | error_report_err(local_err); |
| 652 | } |
| 653 | } |
| 654 | |
| 655 | g_free(migration->data_buffer); |
| 656 | migration->data_buffer = NULL; |
| 657 | migration->precopy_init_size = 0; |
| 658 | migration->precopy_dirty_size = 0; |
| 659 | migration->initial_data_sent = false; |
| 660 | migration->request_switchover_ack = false; |
| 661 | vfio_migration_cleanup(vbasedev); |
| 662 | trace_vfio_save_cleanup(vbasedev->name); |
| 663 | } |
| 664 | |
| 665 | static void vfio_state_pending_sync(VFIODevice *vbasedev) |
| 666 | { |
| 667 | VFIOMigration *migration = vbasedev->migration; |
| 668 | |
| 669 | vfio_query_stop_copy_size(vbasedev); |
| 670 | |
| 671 | if (vfio_device_state_is_precopy(vbasedev)) { |
| 672 | vfio_query_precopy_size(migration); |
| 673 | } |
| 674 | } |
| 675 | |
| 676 | static void vfio_state_pending(void *opaque, MigPendingData *pending, |
| 677 | bool exact, bool final) |
| 678 | { |
| 679 | VFIODevice *vbasedev = opaque; |
| 680 | VFIOMigration *migration = vbasedev->migration; |
| 681 | uint64_t precopy_size, stopcopy_size; |
| 682 | bool request_switchover_ack = false; |
| 683 | |
| 684 | /* |
| 685 | * The final pending query runs during switchover downtime. VFIO does not |
| 686 | * need a fresh device pending-data query then to get the latest dirty |
| 687 | * data, so avoid the extra work and report the cached counters below. |
| 688 | * On the other hand, precopy sync is needed to check if switchover ACK was |
| 689 | * requested, but that's already done during guest stop when device is in |
| 690 | * PRE_COPY state. |
| 691 | */ |
| 692 | if (exact && !final) { |
| 693 | vfio_state_pending_sync(vbasedev); |
| 694 | } |
| 695 | |
| 696 | precopy_size = |
| 697 | migration->precopy_init_size + migration->precopy_dirty_size; |
| 698 | |
| 699 | if (migration->stopcopy_size > precopy_size) { |
| 700 | stopcopy_size = migration->stopcopy_size - precopy_size; |
| 701 | } else { |
| 702 | stopcopy_size = 0; |
| 703 | } |
| 704 | |
| 705 | pending->precopy_bytes += precopy_size; |
| 706 | pending->stopcopy_bytes += stopcopy_size; |
| 707 | if (migration->request_switchover_ack) { |
| 708 | pending->switchover_ack_pending++; |
| 709 | request_switchover_ack = true; |
| 710 | migration->request_switchover_ack = false; |
| 711 | } |
| 712 | |
| 713 | trace_vfio_state_pending(vbasedev->name, migration->stopcopy_size, |
| 714 | migration->precopy_init_size, |
| 715 | migration->precopy_dirty_size, |
| 716 | request_switchover_ack, exact, final); |
| 717 | } |
| 718 | |
| 719 | static bool vfio_is_active_iterate(void *opaque) |
| 720 | { |
| 721 | VFIODevice *vbasedev = opaque; |
| 722 | |
| 723 | return vfio_device_state_is_precopy(vbasedev); |
| 724 | } |
| 725 | |
| 726 | /* |
| 727 | * Note about migration rate limiting: VFIO migration buffer size is currently |
| 728 | * limited to 1MB, so there is no need to check if migration rate exceeded (as |
| 729 | * in the worst case it will exceed by 1MB). However, if the buffer size is |
| 730 | * later changed to a bigger value, migration rate should be enforced here. |
| 731 | */ |
| 732 | static int vfio_save_iterate(QEMUFile *f, void *opaque) |
| 733 | { |
| 734 | VFIODevice *vbasedev = opaque; |
| 735 | VFIOMigration *migration = vbasedev->migration; |
| 736 | ssize_t data_size; |
| 737 | |
| 738 | if (!migration->event_save_iterate_started) { |
| 739 | trace_vfio_save_iterate_start(vbasedev->name); |
| 740 | migration->event_save_iterate_started = true; |
| 741 | } |
| 742 | |
| 743 | data_size = vfio_save_block(f, migration); |
| 744 | if (data_size < 0) { |
| 745 | return data_size; |
| 746 | } |
| 747 | |
| 748 | vfio_update_estimated_pending_data(migration, data_size); |
| 749 | |
| 750 | if (!vfio_send_init_data_flag(f, migration)) { |
| 751 | qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE); |
| 752 | } |
| 753 | |
| 754 | trace_vfio_save_iterate(vbasedev->name, migration->precopy_init_size, |
| 755 | migration->precopy_dirty_size); |
| 756 | |
| 757 | return !migration->precopy_init_size && !migration->precopy_dirty_size; |
| 758 | } |
| 759 | |
| 760 | static int vfio_save_complete_precopy(QEMUFile *f, void *opaque) |
| 761 | { |
| 762 | VFIODevice *vbasedev = opaque; |
| 763 | ssize_t data_size; |
| 764 | int ret; |
| 765 | Error *local_err = NULL; |
| 766 | |
| 767 | if (vfio_multifd_transfer_enabled(vbasedev)) { |
| 768 | vfio_multifd_emit_dummy_eos(vbasedev, f); |
| 769 | return 0; |
| 770 | } |
| 771 | |
| 772 | trace_vfio_save_complete_precopy_start(vbasedev->name); |
| 773 | |
| 774 | /* We reach here with device state STOP or STOP_COPY only */ |
| 775 | ret = vfio_migration_set_state(vbasedev, VFIO_DEVICE_STATE_STOP_COPY, |
| 776 | VFIO_DEVICE_STATE_STOP, &local_err); |
| 777 | if (ret) { |
| 778 | error_report_err(local_err); |
| 779 | return ret; |
| 780 | } |
| 781 | |
| 782 | do { |
| 783 | data_size = vfio_save_block(f, vbasedev->migration); |
| 784 | if (data_size < 0) { |
| 785 | return data_size; |
| 786 | } |
| 787 | } while (data_size); |
| 788 | |
| 789 | qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE); |
| 790 | ret = qemu_file_get_error(f); |
| 791 | |
| 792 | trace_vfio_save_complete_precopy(vbasedev->name, ret); |
| 793 | |
| 794 | return ret; |
| 795 | } |
| 796 | |
| 797 | static void vfio_save_state(QEMUFile *f, void *opaque) |
| 798 | { |
| 799 | VFIODevice *vbasedev = opaque; |
| 800 | Error *local_err = NULL; |
| 801 | int ret; |
| 802 | |
| 803 | if (vfio_multifd_transfer_enabled(vbasedev)) { |
| 804 | if (vfio_load_config_after_iter(vbasedev)) { |
| 805 | qemu_put_be64(f, VFIO_MIG_FLAG_DEV_CONFIG_LOAD_READY); |
| 806 | } else { |
| 807 | vfio_multifd_emit_dummy_eos(vbasedev, f); |
| 808 | } |
| 809 | return; |
| 810 | } |
| 811 | |
| 812 | ret = vfio_save_device_config_state(f, opaque, &local_err); |
| 813 | if (ret) { |
| 814 | error_prepend(&local_err, |
| 815 | "vfio: Failed to save device config space of %s - ", |
| 816 | vbasedev->name); |
| 817 | qemu_file_set_error_obj(f, ret, local_err); |
| 818 | } |
| 819 | } |
| 820 | |
| 821 | static int vfio_load_setup(QEMUFile *f, void *opaque, Error **errp) |
| 822 | { |
| 823 | VFIODevice *vbasedev = opaque; |
| 824 | VFIOMigration *migration = vbasedev->migration; |
| 825 | int ret; |
| 826 | |
| 827 | if (!vfio_multifd_setup(vbasedev, true, errp)) { |
| 828 | return -EINVAL; |
| 829 | } |
| 830 | |
| 831 | ret = vfio_migration_set_state(vbasedev, VFIO_DEVICE_STATE_RESUMING, |
| 832 | migration->device_state, errp); |
| 833 | if (ret) { |
| 834 | return ret; |
| 835 | } |
| 836 | |
| 837 | vfio_request_switchover_ack_legacy(vbasedev); |
| 838 | |
| 839 | return 0; |
| 840 | } |
| 841 | |
| 842 | static int vfio_load_cleanup(void *opaque) |
| 843 | { |
| 844 | VFIODevice *vbasedev = opaque; |
| 845 | |
| 846 | vfio_multifd_cleanup(vbasedev); |
| 847 | |
| 848 | vfio_migration_cleanup(vbasedev); |
| 849 | trace_vfio_load_cleanup(vbasedev->name); |
| 850 | |
| 851 | return 0; |
| 852 | } |
| 853 | |
| 854 | static int vfio_load_state(QEMUFile *f, void *opaque, int version_id) |
| 855 | { |
| 856 | VFIODevice *vbasedev = opaque; |
| 857 | int ret = 0; |
| 858 | uint64_t data; |
| 859 | |
| 860 | data = qemu_get_be64(f); |
| 861 | while (data != VFIO_MIG_FLAG_END_OF_STATE) { |
| 862 | |
| 863 | trace_vfio_load_state(vbasedev->name, data); |
| 864 | |
| 865 | switch (data) { |
| 866 | case VFIO_MIG_FLAG_DEV_CONFIG_STATE: |
| 867 | { |
| 868 | if (vfio_multifd_transfer_enabled(vbasedev)) { |
| 869 | error_report("%s: got DEV_CONFIG_STATE in main migration " |
| 870 | "channel but doing multifd transfer", |
| 871 | vbasedev->name); |
| 872 | return -EINVAL; |
| 873 | } |
| 874 | |
| 875 | return vfio_load_device_config_state(f, opaque); |
| 876 | } |
| 877 | case VFIO_MIG_FLAG_DEV_SETUP_STATE: |
| 878 | { |
| 879 | data = qemu_get_be64(f); |
| 880 | if (data == VFIO_MIG_FLAG_END_OF_STATE) { |
| 881 | return ret; |
| 882 | } else { |
| 883 | error_report("%s: SETUP STATE: EOS not found 0x%"PRIx64, |
| 884 | vbasedev->name, data); |
| 885 | return -EINVAL; |
| 886 | } |
| 887 | break; |
| 888 | } |
| 889 | case VFIO_MIG_FLAG_DEV_DATA_STATE: |
| 890 | { |
| 891 | uint64_t data_size = qemu_get_be64(f); |
| 892 | |
| 893 | if (data_size) { |
| 894 | ret = vfio_load_buffer(f, vbasedev, data_size); |
| 895 | if (ret < 0) { |
| 896 | return ret; |
| 897 | } |
| 898 | } |
| 899 | break; |
| 900 | } |
| 901 | case VFIO_MIG_FLAG_DEV_INIT_DATA_SENT: |
| 902 | { |
| 903 | if (!vfio_precopy_supported(vbasedev) || |
| 904 | !migrate_switchover_ack()) { |
| 905 | error_report("%s: Received INIT_DATA_SENT but switchover ack " |
| 906 | "is not used", vbasedev->name); |
| 907 | return -EINVAL; |
| 908 | } |
| 909 | |
| 910 | ret = qemu_loadvm_approve_switchover(vbasedev->name); |
| 911 | if (ret) { |
| 912 | error_report( |
| 913 | "%s: qemu_loadvm_approve_switchover failed, err=%d (%s)", |
| 914 | vbasedev->name, ret, strerror(-ret)); |
| 915 | } |
| 916 | |
| 917 | return ret; |
| 918 | } |
| 919 | case VFIO_MIG_FLAG_DEV_CONFIG_LOAD_READY: |
| 920 | { |
| 921 | return vfio_load_state_config_load_ready(vbasedev); |
| 922 | } |
| 923 | default: |
| 924 | error_report("%s: Unknown tag 0x%"PRIx64, vbasedev->name, data); |
| 925 | return -EINVAL; |
| 926 | } |
| 927 | |
| 928 | data = qemu_get_be64(f); |
| 929 | ret = qemu_file_get_error(f); |
| 930 | if (ret) { |
| 931 | return ret; |
| 932 | } |
| 933 | } |
| 934 | return ret; |
| 935 | } |
| 936 | |
| 937 | static int vfio_switchover_start(void *opaque) |
| 938 | { |
| 939 | VFIODevice *vbasedev = opaque; |
| 940 | |
| 941 | if (vfio_multifd_transfer_enabled(vbasedev)) { |
| 942 | return vfio_multifd_switchover_start(vbasedev); |
| 943 | } |
| 944 | |
| 945 | return 0; |
| 946 | } |
| 947 | |
| 948 | static const SaveVMHandlers savevm_vfio_handlers = { |
| 949 | .save_prepare = vfio_save_prepare, |
| 950 | .save_setup = vfio_save_setup, |
| 951 | .save_cleanup = vfio_save_cleanup, |
| 952 | .save_query_pending = vfio_state_pending, |
| 953 | .is_active_iterate = vfio_is_active_iterate, |
| 954 | .save_live_iterate = vfio_save_iterate, |
| 955 | .save_complete = vfio_save_complete_precopy, |
| 956 | .save_state = vfio_save_state, |
| 957 | .load_setup = vfio_load_setup, |
| 958 | .load_cleanup = vfio_load_cleanup, |
| 959 | .load_state = vfio_load_state, |
| 960 | /* |
| 961 | * Multifd support |
| 962 | */ |
| 963 | .load_state_buffer = vfio_multifd_load_state_buffer, |
| 964 | .switchover_start = vfio_switchover_start, |
| 965 | .save_complete_precopy_thread = vfio_multifd_save_complete_precopy_thread, |
| 966 | }; |
| 967 | |
| 968 | /* ---------------------------------------------------------------------- */ |
| 969 | |
| 970 | static void vfio_final_precopy_reinit_check(VFIODevice *vbasedev) |
| 971 | { |
| 972 | VFIOMigration *migration = vbasedev->migration; |
| 973 | int ret; |
| 974 | |
| 975 | if (!migration->precopy_info_v2_used || !migrate_switchover_ack() || |
| 976 | migrate_switchover_ack_legacy()) { |
| 977 | return; |
| 978 | } |
| 979 | |
| 980 | ret = vfio_query_precopy_size(migration); |
| 981 | if (ret) { |
| 982 | error_report("%s: Final precopy reinit check failed (err: %d)", |
| 983 | vbasedev->name, ret); |
| 984 | /* If query failed, assume reinit and request switchover-ack */ |
| 985 | migration->request_switchover_ack = true; |
| 986 | migration->initial_data_sent = false; |
| 987 | } |
| 988 | } |
| 989 | |
| 990 | static void vfio_vmstate_change_prepare(void *opaque, bool running, |
| 991 | RunState state) |
| 992 | { |
| 993 | VFIODevice *vbasedev = opaque; |
| 994 | VFIOMigration *migration = vbasedev->migration; |
| 995 | enum vfio_device_mig_state new_state; |
| 996 | Error *local_err = NULL; |
| 997 | int ret; |
| 998 | |
| 999 | new_state = migration->device_state == VFIO_DEVICE_STATE_PRE_COPY ? |
| 1000 | VFIO_DEVICE_STATE_PRE_COPY_P2P : |
| 1001 | VFIO_DEVICE_STATE_RUNNING_P2P; |
| 1002 | |
| 1003 | if (migration->device_state == VFIO_DEVICE_STATE_PRE_COPY) { |
| 1004 | /* |
| 1005 | * Now that vCPUs are stopped, check if new init_bytes are available |
| 1006 | * since switchover decision, to be reported in the final |
| 1007 | * save_query_pending. |
| 1008 | */ |
| 1009 | vfio_final_precopy_reinit_check(vbasedev); |
| 1010 | } |
| 1011 | |
| 1012 | ret = vfio_migration_set_state_or_reset(vbasedev, new_state, &local_err); |
| 1013 | if (ret) { |
| 1014 | /* |
| 1015 | * Migration should be aborted in this case, but vm_state_notify() |
| 1016 | * currently does not support reporting failures. |
| 1017 | */ |
| 1018 | migration_file_set_error(ret, local_err); |
| 1019 | } |
| 1020 | |
| 1021 | trace_vfio_vmstate_change_prepare(vbasedev->name, running, |
| 1022 | RunState_str(state), |
| 1023 | mig_state_to_str(new_state)); |
| 1024 | } |
| 1025 | |
| 1026 | static void vfio_vmstate_change(void *opaque, bool running, RunState state) |
| 1027 | { |
| 1028 | VFIODevice *vbasedev = opaque; |
| 1029 | enum vfio_device_mig_state new_state; |
| 1030 | Error *local_err = NULL; |
| 1031 | int ret; |
| 1032 | |
| 1033 | if (running) { |
| 1034 | new_state = VFIO_DEVICE_STATE_RUNNING; |
| 1035 | } else { |
| 1036 | new_state = |
| 1037 | (vfio_device_state_is_precopy(vbasedev) && |
| 1038 | (state == RUN_STATE_FINISH_MIGRATE || state == RUN_STATE_PAUSED)) ? |
| 1039 | VFIO_DEVICE_STATE_STOP_COPY : |
| 1040 | VFIO_DEVICE_STATE_STOP; |
| 1041 | } |
| 1042 | |
| 1043 | ret = vfio_migration_set_state_or_reset(vbasedev, new_state, &local_err); |
| 1044 | if (ret) { |
| 1045 | /* |
| 1046 | * Migration should be aborted in this case, but vm_state_notify() |
| 1047 | * currently does not support reporting failures. |
| 1048 | */ |
| 1049 | migration_file_set_error(ret, local_err); |
| 1050 | } |
| 1051 | |
| 1052 | trace_vfio_vmstate_change(vbasedev->name, running, RunState_str(state), |
| 1053 | mig_state_to_str(new_state)); |
| 1054 | } |
| 1055 | |
| 1056 | static int vfio_migration_state_notifier(NotifierWithReturn *notifier, |
| 1057 | MigrationEvent *e, Error **errp) |
| 1058 | { |
| 1059 | VFIOMigration *migration = container_of(notifier, VFIOMigration, |
| 1060 | migration_state); |
| 1061 | VFIODevice *vbasedev = migration->vbasedev; |
| 1062 | Error *local_err = NULL; |
| 1063 | int ret; |
| 1064 | |
| 1065 | trace_vfio_migration_state_notifier(vbasedev->name, e->type); |
| 1066 | |
| 1067 | if (e->type == MIG_EVENT_FAILED) { |
| 1068 | /* |
| 1069 | * MigrationNotifyFunc may not return an error code and an Error |
| 1070 | * object for MIG_EVENT_FAILED. Hence, report the error |
| 1071 | * locally and ignore the errp argument. |
| 1072 | */ |
| 1073 | ret = vfio_migration_set_state_or_reset(vbasedev, |
| 1074 | VFIO_DEVICE_STATE_RUNNING, |
| 1075 | &local_err); |
| 1076 | if (ret) { |
| 1077 | error_report_err(local_err); |
| 1078 | } |
| 1079 | } |
| 1080 | return 0; |
| 1081 | } |
| 1082 | |
| 1083 | static void vfio_migration_free(VFIODevice *vbasedev) |
| 1084 | { |
| 1085 | g_free(vbasedev->migration); |
| 1086 | vbasedev->migration = NULL; |
| 1087 | } |
| 1088 | |
| 1089 | static int vfio_migration_query_flags(VFIODevice *vbasedev, uint64_t *mig_flags) |
| 1090 | { |
| 1091 | uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) + |
| 1092 | sizeof(struct vfio_device_feature_migration), |
| 1093 | sizeof(uint64_t))] = {}; |
| 1094 | struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; |
| 1095 | struct vfio_device_feature_migration *mig = |
| 1096 | (struct vfio_device_feature_migration *)feature->data; |
| 1097 | |
| 1098 | feature->argsz = sizeof(buf); |
| 1099 | feature->flags = VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_MIGRATION; |
| 1100 | if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { |
| 1101 | return -errno; |
| 1102 | } |
| 1103 | |
| 1104 | *mig_flags = mig->flags; |
| 1105 | |
| 1106 | return 0; |
| 1107 | } |
| 1108 | |
| 1109 | /* Returns 1 on success, 0 if not supported and negative errno on failure */ |
| 1110 | static int vfio_migration_set_precopy_info_v2(VFIODevice *vbasedev) |
| 1111 | { |
| 1112 | uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature), |
| 1113 | sizeof(uint64_t))] = {}; |
| 1114 | struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; |
| 1115 | |
| 1116 | feature->argsz = sizeof(buf); |
| 1117 | feature->flags = |
| 1118 | VFIO_DEVICE_FEATURE_SET | VFIO_DEVICE_FEATURE_MIG_PRECOPY_INFOv2; |
| 1119 | if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { |
| 1120 | if (errno == ENOTTY) { |
| 1121 | return 0; |
| 1122 | } |
| 1123 | |
| 1124 | return -errno; |
| 1125 | } |
| 1126 | |
| 1127 | return 1; |
| 1128 | } |
| 1129 | |
| 1130 | static bool vfio_dma_logging_supported(VFIODevice *vbasedev) |
| 1131 | { |
| 1132 | uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature), |
| 1133 | sizeof(uint64_t))] = {}; |
| 1134 | struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; |
| 1135 | |
| 1136 | feature->argsz = sizeof(buf); |
| 1137 | feature->flags = VFIO_DEVICE_FEATURE_PROBE | |
| 1138 | VFIO_DEVICE_FEATURE_DMA_LOGGING_START; |
| 1139 | |
| 1140 | return !ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature); |
| 1141 | } |
| 1142 | |
| 1143 | static bool vfio_migration_init(VFIODevice *vbasedev, Error **errp) |
| 1144 | { |
| 1145 | int ret; |
| 1146 | Object *obj; |
| 1147 | VFIOMigration *migration; |
| 1148 | char id[256] = ""; |
| 1149 | g_autofree char *path = NULL, *oid = NULL; |
| 1150 | uint64_t mig_flags = 0; |
| 1151 | bool precopy_info_v2_used = false; |
| 1152 | VMChangeStateHandler *prepare_cb; |
| 1153 | |
| 1154 | if (!vbasedev->ops->vfio_get_object) { |
| 1155 | error_setg(errp, "no vfio_get_object handler"); |
| 1156 | return false; |
| 1157 | } |
| 1158 | |
| 1159 | obj = vbasedev->ops->vfio_get_object(vbasedev); |
| 1160 | if (!obj) { |
| 1161 | error_setg(errp, "failed to get object"); |
| 1162 | return false; |
| 1163 | } |
| 1164 | |
| 1165 | ret = vfio_migration_query_flags(vbasedev, &mig_flags); |
| 1166 | if (ret) { |
| 1167 | if (ret == -ENOTTY) { |
| 1168 | error_setg_errno(errp, -ret, |
| 1169 | "migration is not supported in kernel"); |
| 1170 | } else { |
| 1171 | error_setg_errno(errp, -ret, "failed to query migration flags"); |
| 1172 | } |
| 1173 | |
| 1174 | return false; |
| 1175 | } |
| 1176 | |
| 1177 | /* Basic migration functionality must be supported */ |
| 1178 | if (!(mig_flags & VFIO_MIGRATION_STOP_COPY)) { |
| 1179 | error_setg(errp, "VFIO_MIGRATION_STOP_COPY is not supported"); |
| 1180 | return false; |
| 1181 | } |
| 1182 | |
| 1183 | if (mig_flags & VFIO_MIGRATION_PRE_COPY) { |
| 1184 | ret = vfio_migration_set_precopy_info_v2(vbasedev); |
| 1185 | if (ret < 0) { |
| 1186 | error_setg_errno(errp, -ret, "failed to set precopy info v2"); |
| 1187 | return false; |
| 1188 | } |
| 1189 | precopy_info_v2_used = ret; |
| 1190 | } |
| 1191 | |
| 1192 | vbasedev->migration = g_new0(VFIOMigration, 1); |
| 1193 | migration = vbasedev->migration; |
| 1194 | migration->vbasedev = vbasedev; |
| 1195 | migration->device_state = VFIO_DEVICE_STATE_RUNNING; |
| 1196 | migration->data_fd = -1; |
| 1197 | migration->mig_flags = mig_flags; |
| 1198 | migration->precopy_info_v2_used = precopy_info_v2_used; |
| 1199 | |
| 1200 | vbasedev->dirty_pages_supported = vfio_dma_logging_supported(vbasedev); |
| 1201 | |
| 1202 | oid = vmstate_if_get_id(VMSTATE_IF(DEVICE(obj))); |
| 1203 | if (oid) { |
| 1204 | path = g_strdup_printf("%s/vfio", oid); |
| 1205 | } else { |
| 1206 | path = g_strdup("vfio"); |
| 1207 | } |
| 1208 | strpadcpy(id, sizeof(id), path, '\0'); |
| 1209 | |
| 1210 | register_savevm_live(id, VMSTATE_INSTANCE_ID_ANY, 1, &savevm_vfio_handlers, |
| 1211 | vbasedev); |
| 1212 | |
| 1213 | prepare_cb = migration->mig_flags & VFIO_MIGRATION_P2P ? |
| 1214 | vfio_vmstate_change_prepare : |
| 1215 | NULL; |
| 1216 | migration->vm_state = qdev_add_vm_change_state_handler_full( |
| 1217 | vbasedev->dev, vfio_vmstate_change, prepare_cb, NULL, vbasedev); |
| 1218 | migration_add_notifier(&migration->migration_state, |
| 1219 | vfio_migration_state_notifier); |
| 1220 | |
| 1221 | trace_vfio_migration_init(vbasedev->name, migration->mig_flags, |
| 1222 | migration->precopy_info_v2_used, |
| 1223 | vbasedev->dirty_pages_supported); |
| 1224 | |
| 1225 | return true; |
| 1226 | } |
| 1227 | |
| 1228 | static Error *multiple_devices_migration_blocker; |
| 1229 | |
| 1230 | /* |
| 1231 | * Multiple devices migration is allowed only if all devices support P2P |
| 1232 | * migration. Single device migration is allowed regardless of P2P migration |
| 1233 | * support. |
| 1234 | */ |
| 1235 | static bool vfio_multiple_devices_migration_is_supported(void) |
| 1236 | { |
| 1237 | VFIODevice *vbasedev; |
| 1238 | unsigned int device_num = 0; |
| 1239 | bool all_support_p2p = true; |
| 1240 | |
| 1241 | QLIST_FOREACH(vbasedev, &vfio_device_list, global_next) { |
| 1242 | if (vbasedev->migration) { |
| 1243 | device_num++; |
| 1244 | |
| 1245 | if (!(vbasedev->migration->mig_flags & VFIO_MIGRATION_P2P)) { |
| 1246 | all_support_p2p = false; |
| 1247 | } |
| 1248 | } |
| 1249 | } |
| 1250 | |
| 1251 | return all_support_p2p || device_num <= 1; |
| 1252 | } |
| 1253 | |
| 1254 | static int vfio_block_multiple_devices_migration(VFIODevice *vbasedev, Error **errp) |
| 1255 | { |
| 1256 | if (vfio_multiple_devices_migration_is_supported()) { |
| 1257 | return 0; |
| 1258 | } |
| 1259 | |
| 1260 | if (vbasedev->enable_migration == ON_OFF_AUTO_ON) { |
| 1261 | error_setg(errp, "Multiple VFIO devices migration is supported only if " |
| 1262 | "all of them support P2P migration"); |
| 1263 | return -EINVAL; |
| 1264 | } |
| 1265 | |
| 1266 | if (multiple_devices_migration_blocker) { |
| 1267 | return 0; |
| 1268 | } |
| 1269 | |
| 1270 | error_setg(&multiple_devices_migration_blocker, |
| 1271 | "Multiple VFIO devices migration is supported only if all of " |
| 1272 | "them support P2P migration"); |
| 1273 | return migrate_add_blocker_normal(&multiple_devices_migration_blocker, |
| 1274 | errp); |
| 1275 | } |
| 1276 | |
| 1277 | static void vfio_unblock_multiple_devices_migration(void) |
| 1278 | { |
| 1279 | if (!multiple_devices_migration_blocker || |
| 1280 | !vfio_multiple_devices_migration_is_supported()) { |
| 1281 | return; |
| 1282 | } |
| 1283 | |
| 1284 | migrate_del_blocker(&multiple_devices_migration_blocker); |
| 1285 | } |
| 1286 | |
| 1287 | static void vfio_migration_deinit(VFIODevice *vbasedev) |
| 1288 | { |
| 1289 | VFIOMigration *migration = vbasedev->migration; |
| 1290 | |
| 1291 | migration_remove_notifier(&migration->migration_state); |
| 1292 | qemu_del_vm_change_state_handler(migration->vm_state); |
| 1293 | unregister_savevm(VMSTATE_IF(vbasedev->dev), "vfio", vbasedev); |
| 1294 | vfio_migration_free(vbasedev); |
| 1295 | vfio_unblock_multiple_devices_migration(); |
| 1296 | } |
| 1297 | |
| 1298 | static int vfio_block_migration(VFIODevice *vbasedev, Error *err, Error **errp) |
| 1299 | { |
| 1300 | if (vbasedev->enable_migration == ON_OFF_AUTO_ON) { |
| 1301 | error_propagate(errp, err); |
| 1302 | return -EINVAL; |
| 1303 | } |
| 1304 | |
| 1305 | vbasedev->migration_blocker = error_copy(err); |
| 1306 | error_free(err); |
| 1307 | |
| 1308 | return migrate_add_blocker_normal(&vbasedev->migration_blocker, errp); |
| 1309 | } |
| 1310 | |
| 1311 | /* ---------------------------------------------------------------------- */ |
| 1312 | |
| 1313 | int64_t vfio_migration_bytes_transferred(void) |
| 1314 | { |
| 1315 | return MIN(qatomic_read(&bytes_transferred), INT64_MAX); |
| 1316 | } |
| 1317 | |
| 1318 | void vfio_migration_reset_bytes_transferred(void) |
| 1319 | { |
| 1320 | qatomic_set(&bytes_transferred, 0); |
| 1321 | } |
| 1322 | |
| 1323 | void vfio_migration_add_bytes_transferred(unsigned long val) |
| 1324 | { |
| 1325 | qatomic_add(&bytes_transferred, val); |
| 1326 | } |
| 1327 | |
| 1328 | bool vfio_migration_active(void) |
| 1329 | { |
| 1330 | VFIODevice *vbasedev; |
| 1331 | |
| 1332 | if (QLIST_EMPTY(&vfio_device_list)) { |
| 1333 | return false; |
| 1334 | } |
| 1335 | |
| 1336 | QLIST_FOREACH(vbasedev, &vfio_device_list, global_next) { |
| 1337 | if (vbasedev->migration_blocker) { |
| 1338 | return false; |
| 1339 | } |
| 1340 | } |
| 1341 | return true; |
| 1342 | } |
| 1343 | |
| 1344 | static bool vfio_viommu_preset(VFIODevice *vbasedev) |
| 1345 | { |
| 1346 | return vbasedev->bcontainer->space->as != &address_space_memory; |
| 1347 | } |
| 1348 | |
| 1349 | static bool vfio_dirty_tracking_exceed_limit(VFIODevice *vbasedev) |
| 1350 | { |
| 1351 | VFIOContainer *bcontainer = vbasedev->bcontainer; |
| 1352 | uint64_t max_size, page_size; |
| 1353 | |
| 1354 | if (!bcontainer->dirty_pages_supported) { |
| 1355 | return false; |
| 1356 | } |
| 1357 | |
| 1358 | /* |
| 1359 | * VFIO IOMMU type1 driver has limitation of bitmap size on unmap_bitmap |
| 1360 | * ioctl(), calculate the limit and compare with guest memory size to |
| 1361 | * catch dirty tracking failure early. |
| 1362 | * |
| 1363 | * This limit is 8TB with default kernel and QEMU config, we are a bit |
| 1364 | * conservative here as VM memory layout may be nonconsecutive or VM |
| 1365 | * can run with vIOMMU enabled so the limitation could be relaxed. One |
| 1366 | * can also switch to use IOMMUFD backend if there is a need to migrate |
| 1367 | * large VM. |
| 1368 | */ |
| 1369 | page_size = 1ULL << ctz64(bcontainer->dirty_pgsizes); |
| 1370 | max_size = bcontainer->max_dirty_bitmap_size * BITS_PER_BYTE * page_size; |
| 1371 | |
| 1372 | return current_machine->ram_size > max_size; |
| 1373 | } |
| 1374 | |
| 1375 | /* |
| 1376 | * Return true when either migration initialized or blocker registered. |
| 1377 | * Currently only return false when adding blocker fails which will |
| 1378 | * de-register vfio device. |
| 1379 | */ |
| 1380 | bool vfio_migration_realize(VFIODevice *vbasedev, Error **errp) |
| 1381 | { |
| 1382 | Error *err = NULL; |
| 1383 | int ret; |
| 1384 | |
| 1385 | if (vbasedev->enable_migration == ON_OFF_AUTO_OFF) { |
| 1386 | error_setg(&err, "%s: Migration is disabled for VFIO device", |
| 1387 | vbasedev->name); |
| 1388 | return !vfio_block_migration(vbasedev, err, errp); |
| 1389 | } |
| 1390 | |
| 1391 | if (!vfio_migration_init(vbasedev, &err)) { |
| 1392 | error_prepend(&err, "%s: VFIO migration init failed: ", vbasedev->name); |
| 1393 | return !vfio_block_migration(vbasedev, err, errp); |
| 1394 | } |
| 1395 | |
| 1396 | if (vfio_device_dirty_pages_disabled(vbasedev) && |
| 1397 | !vbasedev->iommu_dirty_tracking) { |
| 1398 | if (vbasedev->enable_migration == ON_OFF_AUTO_AUTO) { |
| 1399 | error_setg(&err, |
| 1400 | "%s: VFIO device doesn't support device and " |
| 1401 | "IOMMU dirty tracking", vbasedev->name); |
| 1402 | goto add_blocker; |
| 1403 | } |
| 1404 | |
| 1405 | if (vfio_dirty_tracking_exceed_limit(vbasedev)) { |
| 1406 | error_setg(&err, "%s: Migration is currently not supported with " |
| 1407 | "large memory VM due to dirty tracking limitation in " |
| 1408 | "backend", vbasedev->name); |
| 1409 | goto add_blocker; |
| 1410 | } |
| 1411 | |
| 1412 | warn_report("%s: VFIO device doesn't support device and " |
| 1413 | "IOMMU dirty tracking", vbasedev->name); |
| 1414 | } |
| 1415 | |
| 1416 | ret = vfio_block_multiple_devices_migration(vbasedev, errp); |
| 1417 | if (ret) { |
| 1418 | goto out_deinit; |
| 1419 | } |
| 1420 | |
| 1421 | if (!vfio_device_dirty_pages_disabled(vbasedev) && |
| 1422 | vfio_viommu_preset(vbasedev)) { |
| 1423 | error_setg(&err, "%s: Migration is currently not supported " |
| 1424 | "with vIOMMU enabled", vbasedev->name); |
| 1425 | goto add_blocker; |
| 1426 | } |
| 1427 | |
| 1428 | trace_vfio_migration_realize(vbasedev->name); |
| 1429 | return true; |
| 1430 | |
| 1431 | add_blocker: |
| 1432 | ret = vfio_block_migration(vbasedev, err, errp); |
| 1433 | out_deinit: |
| 1434 | if (ret) { |
| 1435 | vfio_migration_deinit(vbasedev); |
| 1436 | } |
| 1437 | return !ret; |
| 1438 | } |
| 1439 | |
| 1440 | void vfio_migration_exit(VFIODevice *vbasedev) |
| 1441 | { |
| 1442 | if (vbasedev->migration) { |
| 1443 | vfio_migration_deinit(vbasedev); |
| 1444 | } |
| 1445 | |
| 1446 | migrate_del_blocker(&vbasedev->migration_blocker); |
| 1447 | } |
| 1448 | |
| 1449 | bool vfio_device_state_is_running(VFIODevice *vbasedev) |
| 1450 | { |
| 1451 | VFIOMigration *migration = vbasedev->migration; |
| 1452 | |
| 1453 | return migration->device_state == VFIO_DEVICE_STATE_RUNNING || |
| 1454 | migration->device_state == VFIO_DEVICE_STATE_RUNNING_P2P; |
| 1455 | } |
| 1456 | |
| 1457 | bool vfio_device_state_is_precopy(VFIODevice *vbasedev) |
| 1458 | { |
| 1459 | VFIOMigration *migration = vbasedev->migration; |
| 1460 | |
| 1461 | return migration->device_state == VFIO_DEVICE_STATE_PRE_COPY || |
| 1462 | migration->device_state == VFIO_DEVICE_STATE_PRE_COPY_P2P; |
| 1463 | } |