@samitouri / QOSamiQemu / commits / a181df93bb

migration/cpr: use hashtable for cpr fds

Use a GHashTable to store cpr fds to reduce the time consumption of `cpr_find_fd` in scenarios with a large number of fds. The time complexity for `cpr_find_fd` is reduced from O(N) to O(1). Keep cpr fds lookups in a GHashTable during normal runtime while preserving the existing QLIST migration ABI. Build a temporary QLIST from the hash table in pre_save and rebuild the hash table from the loaded QLIST in post_load. To demonstrate the performance improvement, we tested the total time consumed by `cpr_find_fd` (called N times for N fds) under our real-world business scenarios with different numbers of file descriptors. The results are measured in nanoseconds: | Number of FDs | Total time with QLIST (ns) | Total time with GHashTable (ns) | |---------------|----------------------------|---------------------------------| | 540 | 936,753 | 393,358 | | 2,870 | 24,102,342 | 2,212,113 | | 7,530 | 152,715,916 | 5,474,310 | As shown in the data, the lookup time grows exponentially with the QLIST as the number of fds increases. With the GHashTable, the time consumption remains linear (O(1) per lookup), significantly reducing the downtime during the CPR process. Signed-off-by: hongmianquan <hongmianquan@bytedance.com> Link: https://lore.kernel.org/r/20260519134315.27997-1-hongmianquan@bytedance.com Signed-off-by: Peter Xu <peterx@redhat.com>

hongmianquan committed May 19, 2026 at 21:43 UTC a181df93bb32847fad4de6d76360ea0006924b6c
1 file changed +98 -18
migration/cpr.c
+98 -18
@@ -24,6 +24,7 @@
24 /* cpr state container for all information to be saved. */
25
26 CprState cpr_state;
27 +static GHashTable *cpr_fds_hash;
28
29 /****************************************************************************/
30
@@ -48,6 +49,84 @@ static const VMStateDescription vmstate_cpr_fd = {
49 }
50 };
51
52 +static guint cpr_fd_hash(gconstpointer v)
53 +{
54 + const CprFd *elem = v;
55 +
56 + return g_str_hash(elem->name) ^ elem->id;
57 +}
58 +
59 +static gboolean cpr_fd_equal(gconstpointer a, gconstpointer b)
60 +{
61 + const CprFd *elem_a = a;
62 + const CprFd *elem_b = b;
63 +
64 + return !strcmp(elem_a->name, elem_b->name) && elem_a->id == elem_b->id;
65 +}
66 +
67 +static void cpr_fd_destroy(gpointer data)
68 +{
69 + CprFd *elem = data;
70 +
71 + g_free(elem->name);
72 + g_free(elem);
73 +}
74 +
75 +static GHashTable *get_cpr_fds_hash(void)
76 +{
77 + if (!cpr_fds_hash) {
78 + cpr_fds_hash = g_hash_table_new_full(cpr_fd_hash, cpr_fd_equal,
79 + cpr_fd_destroy, NULL);
80 + }
81 +
82 + return cpr_fds_hash;
83 +}
84 +
85 +static void cpr_fd_hash_insert(CprFd *elem)
86 +{
87 + /* Use the same CprFd as key and value. */
88 + g_hash_table_insert(get_cpr_fds_hash(), elem, elem);
89 +}
90 +
91 +static int cpr_fd_pre_save(void *opaque)
92 +{
93 + CprState *state = (CprState *)opaque;
94 + GHashTableIter iter;
95 + CprFd *elem;
96 +
97 + QLIST_INIT(&state->fds);
98 +
99 + g_hash_table_iter_init(&iter, get_cpr_fds_hash());
100 + while (g_hash_table_iter_next(&iter, (gpointer *)&elem, NULL)) {
101 + QLIST_INSERT_HEAD(&state->fds, elem, next);
102 + }
103 +
104 + return 0;
105 +}
106 +
107 +static int cpr_fd_post_load(void *opaque, int version_id)
108 +{
109 + CprState *state = (CprState *)opaque;
110 + CprFd *elem;
111 +
112 + while ((elem = QLIST_FIRST(&state->fds))) {
113 + QLIST_REMOVE(elem, next);
114 +
115 + /*
116 + * Preserve legacy QLIST lookup semantics if duplicate keys exist in
117 + * the incoming stream: the first matching entry wins.
118 + */
119 + if (g_hash_table_contains(get_cpr_fds_hash(), elem)) {
120 + cpr_fd_destroy(elem);
121 + continue;
122 + }
123 +
124 + cpr_fd_hash_insert(elem);
125 + }
126 +
127 + return 0;
128 +}
129 +
130 void cpr_save_fd(const char *name, int id, int fd)
131 {
132 CprFd *elem = g_new0(CprFd, 1);
@@ -57,37 +136,34 @@ void cpr_save_fd(const char *name, int id, int fd)
136 elem->namelen = strlen(name) + 1;
137 elem->id = id;
138 elem->fd = fd;
60 - QLIST_INSERT_HEAD(&cpr_state.fds, elem, next);
139 + cpr_fd_hash_insert(elem);
140 }
141
63 -static CprFd *find_fd(CprFdList *head, const char *name, int id)
142 +static CprFd *find_fd(const char *name, int id)
143 {
65 - CprFd *elem;
144 + CprFd key = {
145 + .name = (char *)name,
146 + .id = id,
147 + };
148
67 - QLIST_FOREACH(elem, head, next) {
68 - if (!strcmp(elem->name, name) && elem->id == id) {
69 - return elem;
70 - }
71 - }
72 - return NULL;
149 + return g_hash_table_lookup(get_cpr_fds_hash(), &key);
150 }
151
152 void cpr_delete_fd(const char *name, int id)
153 {
77 - CprFd *elem = find_fd(&cpr_state.fds, name, id);
154 + CprFd key = {
155 + .name = (char *)name,
156 + .id = id,
157 + };
158
79 - if (elem) {
80 - QLIST_REMOVE(elem, next);
81 - g_free(elem->name);
82 - g_free(elem);
83 - }
159 + g_hash_table_remove(get_cpr_fds_hash(), &key);
160
161 trace_cpr_delete_fd(name, id);
162 }
163
164 int cpr_find_fd(const char *name, int id)
165 {
90 - CprFd *elem = find_fd(&cpr_state.fds, name, id);
166 + CprFd *elem = find_fd(name, id);
167 int fd = elem ? elem->fd : -1;
168
169 trace_cpr_find_fd(name, id, fd);
@@ -96,7 +172,7 @@ int cpr_find_fd(const char *name, int id)
172
173 void cpr_resave_fd(const char *name, int id, int fd)
174 {
99 - CprFd *elem = find_fd(&cpr_state.fds, name, id);
175 + CprFd *elem = find_fd(name, id);
176 int old_fd = elem ? elem->fd : -1;
177
178 if (old_fd < 0) {
@@ -125,9 +201,11 @@ int cpr_open_fd(const char *path, int flags, const char *name, int id,
201
202 bool cpr_walk_fd(cpr_walk_fd_cb cb)
203 {
204 + GHashTableIter iter;
205 CprFd *elem;
206
130 - QLIST_FOREACH(elem, &cpr_state.fds, next) {
207 + g_hash_table_iter_init(&iter, get_cpr_fds_hash());
208 + while (g_hash_table_iter_next(&iter, (gpointer *)&elem, NULL)) {
209 g_assert(elem->fd >= 0);
210 if (!cb(elem->fd)) {
211 return false;
@@ -141,6 +219,8 @@ static const VMStateDescription vmstate_cpr_state = {
219 .name = CPR_STATE,
220 .version_id = 1,
221 .minimum_version_id = 1,
222 + .pre_save = cpr_fd_pre_save,
223 + .post_load = cpr_fd_post_load,
224 .fields = (VMStateField[]) {
225 VMSTATE_QLIST_V(fds, CprState, 1, vmstate_cpr_fd, CprFd, next),
226 VMSTATE_END_OF_LIST()