migration: add support for fault thread to load pages from disk
In fast snapshot load, we would like to serve faults as soon as possible hence loading pages directly instead of requesting a source Add postcopy_mapped_ram_load_page() function which serves single page fault. It uses bitmap_test_and_clear_atomic() on pending_bmap to prevent multiple threads from loading same page. It loads a page or pages depending on size of guest pages and host pages, loading exactly the larger of two. If the entire page is zero postcopy_place_page_zero() is used for efficiency and in case some part is non zero it is part by part loaded on loop using new function postcopy_mapped_ram_load_guest_page() which loads a single guest page in a buffer which is then placed using postcopy_place_page(). This covers all possible cases for various page sizes of host and guest. Because postcopy_place_page and in general postcopy does not support larger guest pages this will not work in case of guest pages sizes larger than host. Update postcopy_ram_fault_thread to call postcopy_mapped_ram_load_page instead of requesting source in case of fast snapshot load. to_src_file check is bypassed in fast snapshot load case as there is no source. Call try_mark_postcopy_blocktime_begin on every page fault to support postcopy-blocktime. Allocate another channel in postcopy_temp_pages_setup(like the preempt case), for both the fault thread and eager thread to load pages independently. Signed-off-by: Aadeshveer Singh <aadeshveer07@gmail.com> Signed-off-by: Peter Xu <peterx@redhat.com> Signed-off-by: Fabiano Rosas <farosas@suse.de>
Aadeshveer Singh committed
Aug 16, 2026 at 23:16 UTC
a55fef7c65cbfa16608d5a6c052faf4d8de54f49
1 file changed
+166
-17
migration/postcopy-ram.c
+166
-17
@@ -946,6 +946,130 @@ int postcopy_wake_shared(struct PostCopyFD *pcfd,
946
pagesize);
947
}
948
949
+/*
950
+ * Load a single guest page from source file into the buffer.
951
+ * NOTE: This is not an atomic operation and should not be used to directly load
952
+ * pages on page faults in postcopy. It is meant to fill in buffer that can then
953
+ * be copied into the faulting location using UFFDIO_COPY.
954
+ */
955
+static bool postcopy_mapped_ram_load_guest_page(MigrationIncomingState *mis,
956
+ RAMBlock *rb,
957
+ ram_addr_t rb_offset, void *buf,
958
+ Error **errp)
959
+{
960
+ ERRP_GUARD();
961
+ size_t page = rb_offset / qemu_target_page_size();
962
+ size_t read;
963
+
964
+ if (test_bit(page, rb->file_bmap)) {
965
+ /*
966
+ * This can happen concurrently, but it's thread-safe because
967
+ * qemu_get_buffer_at() is thread-safe, and the caller will be using
968
+ * different temporary buffers.
969
+ */
970
+ read =
971
+ qemu_get_buffer_at(mis->from_src_file, buf, qemu_target_page_size(),
972
+ rb->pages_offset + rb_offset, errp);
973
+
974
+ if (read != qemu_target_page_size()) {
975
+ error_prepend(errp,
976
+ "Could not read page %zu from RAM Block %s: ", page,
977
+ rb->idstr);
978
+ return false;
979
+ }
980
+ } else {
981
+ memset(buf, '\0', qemu_target_page_size());
982
+ }
983
+ return true;
984
+}
985
+
986
+/**
987
+ * postcopy_mapped_ram_load_page() - Load pages required to access host address.
988
+ * @mis: Migration Incoming State.
989
+ * @rb: RAMBlock from where page is loaded.
990
+ * @rb_offset: Offset of target page in RAMBlock.
991
+ * @haddr: Base of target page where to load in page.
992
+ * @channel: Used to identify between threads and use corresponding temp.
993
+ * @errp: Set error in case of failure
994
+ *
995
+ * Load page(s) from RAMBlock covering the faulting address. We might need to
996
+ * load multiple pages in the case when host page size is greater than guest
997
+ * page size. As userfaultfd works on granularity of host pages, we might need
998
+ * to load guest pages in single operation.
999
+ *
1000
+ * Return: True on success.
1001
+ */
1002
+static bool postcopy_mapped_ram_load_page(MigrationIncomingState *mis,
1003
+ RAMBlock *rb, ram_addr_t rb_offset,
1004
+ uint64_t haddr, int channel,
1005
+ Error **errp)
1006
+{
1007
+ void *place_source = mis->postcopy_tmp_pages[channel].tmp_huge_page;
1008
+ char *buffer_ptr = (char *)place_source;
1009
+ size_t guest_pages_to_load =
1010
+ MAX(1, qemu_ram_pagesize(rb) / qemu_target_page_size());
1011
+ size_t guest_page;
1012
+ size_t host_page;
1013
+
1014
+ /*
1015
+ * If guest page size is greater than host page size uffd needs to load one
1016
+ * guest page and multiple host pages, hence the offsets need to aligned
1017
+ * with guest pages (which is automatically aligned with host pages). In the
1018
+ * same case we need to check range of bits on pending_bmap(bit per host
1019
+ * page) to decide whether all the page have been loaded.
1020
+ *
1021
+ * NOTE: This is future proofing as currently target page size greater than
1022
+ * host page size is not supported. However if postcopy does support this in
1023
+ * future, with updated place page functions this function should work
1024
+ * readily.
1025
+ */
1026
+ rb_offset = ROUND_DOWN(rb_offset, qemu_target_page_size());
1027
+ haddr = ROUND_DOWN(haddr, qemu_target_page_size());
1028
+ guest_page = rb_offset >> qemu_target_page_bits();
1029
+ host_page = rb_offset / qemu_ram_pagesize(rb);
1030
+
1031
+ /*
1032
+ * pending_bmap needs the index of host or guest page based on which is
1033
+ * larger. As page index is inversely proportional to page size we use the
1034
+ * minimum of both.
1035
+ */
1036
+ if (bitmap_test_and_clear_atomic(rb->pending_bmap,
1037
+ MIN(host_page, guest_page), 1)) {
1038
+ if (find_next_bit(rb->file_bmap, guest_page + guest_pages_to_load,
1039
+ guest_page) == guest_page + guest_pages_to_load) {
1040
+ /* It is efficient to use UFFDIO_ZERO if all pages are zero */
1041
+ if (postcopy_place_page_zero(mis, (void *)haddr, rb)) {
1042
+ error_setg(errp,
1043
+ "Failed to place zero page %zu from RAM Block %s at "
1044
+ "address %" PRIu64,
1045
+ guest_page, rb->idstr, haddr);
1046
+ return false;
1047
+ }
1048
+ } else {
1049
+ size_t load_size = guest_pages_to_load * qemu_target_page_size();
1050
+ size_t offset;
1051
+
1052
+ for (offset = 0; offset < load_size;
1053
+ offset += qemu_target_page_size()) {
1054
+ if (!postcopy_mapped_ram_load_guest_page(
1055
+ mis, rb, rb_offset + offset, buffer_ptr + offset,
1056
+ errp)) {
1057
+ return false;
1058
+ }
1059
+ }
1060
+
1061
+ if (postcopy_place_page(mis, (void *)haddr, place_source, rb)) {
1062
+ error_setg(errp,
1063
+ "Failed to place page %zu from RAM Block %s at "
1064
+ "address %" PRIu64,
1065
+ guest_page, rb->idstr, haddr);
1066
+ return false;
1067
+ }
1068
+ }
1069
+ }
1070
+ return true;
1071
+}
1072
+
1073
/*
1074
* NOTE: @tid is only used when postcopy-blocktime feature is enabled, and
1075
* also optional: when zero is provided, the fault accounting will be ignored.
@@ -1308,6 +1432,7 @@ static void *postcopy_ram_fault_thread(void *opaque)
1432
int ret;
1433
size_t index;
1434
RAMBlock *rb = NULL;
1435
+ Error *local_err = NULL;
1436
1437
trace_postcopy_ram_fault_thread_entry();
1438
rcu_register_thread();
@@ -1349,11 +1474,13 @@ static void *postcopy_ram_fault_thread(void *opaque)
1474
break;
1475
}
1476
1352
- if (!mis->to_src_file) {
1477
+ if (!migrate_mapped_ram() && !mis->to_src_file) {
1478
/*
1354
- * Possibly someone tells us that the return path is
1355
- * broken already using the event. We should hold until
1356
- * the channel is rebuilt.
1479
+ * Possibly someone tells us that the return path is broken already
1480
+ * using the event. We should hold until the channel is rebuilt.
1481
+ * Fast snapshot load doesn't support pause and recover, because
1482
+ * it's not necessary: we can fail right away when QEMU just booted
1483
+ * with nothing to lose.
1484
*/
1485
postcopy_pause_fault_thread(mis);
1486
}
@@ -1416,18 +1543,37 @@ static void *postcopy_ram_fault_thread(void *opaque)
1543
qemu_ram_get_idstr(rb),
1544
rb_offset,
1545
msg.arg.pagefault.feat.ptid);
1546
+
1547
+ if (migrate_mapped_ram()) {
1548
+ /* Load page directly in case of fast snapshot load */
1549
+
1550
+ uintptr_t aligned = (uintptr_t)ROUND_DOWN(
1551
+ msg.arg.pagefault.address, qemu_ram_pagesize(rb));
1552
+
1553
+ if (try_mark_postcopy_blocktime_begin(
1554
+ mis, rb, rb_offset, (uintptr_t)aligned,
1555
+ msg.arg.pagefault.feat.ptid)) {
1556
+ if (!postcopy_mapped_ram_load_page(
1557
+ mis, rb, rb_offset, aligned, RAM_CHANNEL_POSTCOPY,
1558
+ &local_err)) {
1559
+ error_report_err(local_err);
1560
+ break;
1561
+ }
1562
+ }
1563
+ } else {
1564
retry:
1420
- /*
1421
- * Send the request to the source - we want to request one
1422
- * of our host page sizes (which is >= TPS)
1423
- */
1424
- ret = postcopy_request_page(mis, rb, rb_offset,
1425
- msg.arg.pagefault.address,
1426
- msg.arg.pagefault.feat.ptid);
1427
- if (ret) {
1428
- /* May be network failure, try to wait for recovery */
1429
- postcopy_pause_fault_thread(mis);
1430
- goto retry;
1565
+ /*
1566
+ * Send the request to the source - we want to request one
1567
+ * of our host page sizes (which is >= TPS)
1568
+ */
1569
+ ret = postcopy_request_page(mis, rb, rb_offset,
1570
+ msg.arg.pagefault.address,
1571
+ msg.arg.pagefault.feat.ptid);
1572
+ if (ret) {
1573
+ /* May be network failure, try to wait for recovery */
1574
+ postcopy_pause_fault_thread(mis);
1575
+ goto retry;
1576
+ }
1577
}
1578
}
1579
@@ -1499,8 +1645,11 @@ static int postcopy_temp_pages_setup(MigrationIncomingState *mis, Error **errp)
1645
unsigned i, channels;
1646
void *temp_page;
1647
1502
- if (migrate_postcopy_preempt()) {
1503
- /* If preemption enabled, need extra channel for urgent requests */
1648
+ if (migrate_postcopy_preempt() || migrate_mapped_ram()) {
1649
+ /*
1650
+ * If preemption enabled or it is fast snapshot load, need extra channel
1651
+ * for urgent requests/faults
1652
+ */
1653
mis->postcopy_channels = RAM_CHANNEL_MAX;
1654
} else {
1655
/* Both precopy/postcopy on the same channel */