| 1 | /* |
| 2 | * Multifd UADK compression accelerator implementation |
| 3 | * |
| 4 | * Copyright (c) 2024 Huawei Technologies R & D (UK) Ltd |
| 5 | * |
| 6 | * Authors: |
| 7 | * Shameer Kolothum <shameerali.kolothum.thodi@huawei.com> |
| 8 | * |
| 9 | * This work is licensed under the terms of the GNU GPL, version 2 or later. |
| 10 | * See the COPYING file in the top-level directory. |
| 11 | */ |
| 12 | |
| 13 | #include "qemu/osdep.h" |
| 14 | #include "qemu/module.h" |
| 15 | #include "qapi/error.h" |
| 16 | #include "system/ramblock.h" |
| 17 | #include "migration.h" |
| 18 | #include "multifd.h" |
| 19 | #include "options.h" |
| 20 | #include "qemu/error-report.h" |
| 21 | #include "uadk/wd_comp.h" |
| 22 | #include "uadk/wd_sched.h" |
| 23 | |
| 24 | struct wd_data { |
| 25 | handle_t handle; |
| 26 | uint8_t *buf; |
| 27 | uint32_t *buf_hdr; |
| 28 | }; |
| 29 | |
| 30 | static bool uadk_hw_init(void) |
| 31 | { |
| 32 | char alg[] = "zlib"; |
| 33 | int ret; |
| 34 | |
| 35 | ret = wd_comp_init2(alg, SCHED_POLICY_RR, TASK_HW); |
| 36 | if (ret && ret != -WD_EEXIST) { |
| 37 | return false; |
| 38 | } else { |
| 39 | return true; |
| 40 | } |
| 41 | } |
| 42 | |
| 43 | static struct wd_data *multifd_uadk_init_sess(uint32_t count, |
| 44 | uint32_t page_size, |
| 45 | bool compress, Error **errp) |
| 46 | { |
| 47 | struct wd_comp_sess_setup ss = {0}; |
| 48 | struct sched_params param = {0}; |
| 49 | uint32_t size = count * page_size; |
| 50 | struct wd_data *wd; |
| 51 | |
| 52 | wd = g_new0(struct wd_data, 1); |
| 53 | |
| 54 | if (uadk_hw_init()) { |
| 55 | ss.alg_type = WD_ZLIB; |
| 56 | if (compress) { |
| 57 | ss.op_type = WD_DIR_COMPRESS; |
| 58 | /* Add an additional page for handling output > input */ |
| 59 | size += page_size; |
| 60 | } else { |
| 61 | ss.op_type = WD_DIR_DECOMPRESS; |
| 62 | } |
| 63 | /* We use default level 1 compression and 4K window size */ |
| 64 | param.type = ss.op_type; |
| 65 | ss.sched_param = ¶m; |
| 66 | |
| 67 | wd->handle = wd_comp_alloc_sess(&ss); |
| 68 | if (!wd->handle) { |
| 69 | error_setg(errp, "multifd: failed wd_comp_alloc_sess"); |
| 70 | goto out; |
| 71 | } |
| 72 | } else { |
| 73 | /* For CI test use */ |
| 74 | warn_report_once("UADK hardware not available. Switch to no compression mode"); |
| 75 | } |
| 76 | |
| 77 | wd->buf = g_try_malloc(size); |
| 78 | if (!wd->buf) { |
| 79 | error_setg(errp, "multifd: out of mem for uadk buf"); |
| 80 | goto out_free_sess; |
| 81 | } |
| 82 | wd->buf_hdr = g_new0(uint32_t, count); |
| 83 | return wd; |
| 84 | |
| 85 | out_free_sess: |
| 86 | if (wd->handle) { |
| 87 | wd_comp_free_sess(wd->handle); |
| 88 | } |
| 89 | out: |
| 90 | wd_comp_uninit2(); |
| 91 | g_free(wd); |
| 92 | return NULL; |
| 93 | } |
| 94 | |
| 95 | static void multifd_uadk_uninit_sess(struct wd_data *wd) |
| 96 | { |
| 97 | if (wd->handle) { |
| 98 | wd_comp_free_sess(wd->handle); |
| 99 | } |
| 100 | wd_comp_uninit2(); |
| 101 | g_free(wd->buf); |
| 102 | g_free(wd->buf_hdr); |
| 103 | g_free(wd); |
| 104 | } |
| 105 | |
| 106 | static int multifd_uadk_send_setup(MultiFDSendParams *p, Error **errp) |
| 107 | { |
| 108 | struct wd_data *wd; |
| 109 | uint32_t page_size = multifd_ram_page_size(); |
| 110 | uint32_t page_count = multifd_ram_page_count(); |
| 111 | |
| 112 | wd = multifd_uadk_init_sess(page_count, page_size, true, errp); |
| 113 | if (!wd) { |
| 114 | return -1; |
| 115 | } |
| 116 | |
| 117 | p->compress_data = wd; |
| 118 | assert(p->iov == NULL); |
| 119 | /* |
| 120 | * Each page will be compressed independently and sent using an IOV. The |
| 121 | * additional two IOVs are used to store packet header and compressed data |
| 122 | * length |
| 123 | */ |
| 124 | |
| 125 | p->iov = g_new0(struct iovec, page_count + 2); |
| 126 | return 0; |
| 127 | } |
| 128 | |
| 129 | static void multifd_uadk_send_cleanup(MultiFDSendParams *p, Error **errp) |
| 130 | { |
| 131 | struct wd_data *wd = p->compress_data; |
| 132 | |
| 133 | multifd_uadk_uninit_sess(wd); |
| 134 | p->compress_data = NULL; |
| 135 | g_free(p->iov); |
| 136 | p->iov = NULL; |
| 137 | } |
| 138 | |
| 139 | static inline void prepare_next_iov(MultiFDSendParams *p, void *base, |
| 140 | uint32_t len) |
| 141 | { |
| 142 | p->iov[p->iovs_num].iov_base = (uint8_t *)base; |
| 143 | p->iov[p->iovs_num].iov_len = len; |
| 144 | p->next_packet_size += len; |
| 145 | p->iovs_num++; |
| 146 | } |
| 147 | |
| 148 | static int multifd_uadk_send_prepare(MultiFDSendParams *p, Error **errp) |
| 149 | { |
| 150 | struct wd_data *uadk_data = p->compress_data; |
| 151 | uint32_t hdr_size; |
| 152 | uint32_t page_size = multifd_ram_page_size(); |
| 153 | uint8_t *buf = uadk_data->buf; |
| 154 | int ret = 0; |
| 155 | MultiFDPages_t *pages = &p->data->u.ram; |
| 156 | |
| 157 | if (!multifd_send_prepare_common(p)) { |
| 158 | goto out; |
| 159 | } |
| 160 | |
| 161 | hdr_size = pages->normal_num * sizeof(uint32_t); |
| 162 | /* prepare the header that stores the lengths of all compressed data */ |
| 163 | prepare_next_iov(p, uadk_data->buf_hdr, hdr_size); |
| 164 | |
| 165 | for (int i = 0; i < pages->normal_num; i++) { |
| 166 | struct wd_comp_req creq = { |
| 167 | .op_type = WD_DIR_COMPRESS, |
| 168 | .src = pages->block->host + pages->offset[i], |
| 169 | .src_len = page_size, |
| 170 | .dst = buf, |
| 171 | /* Set dst_len to double the src in case compressed out >= page_size */ |
| 172 | .dst_len = page_size * 2, |
| 173 | }; |
| 174 | |
| 175 | if (uadk_data->handle) { |
| 176 | ret = wd_do_comp_sync(uadk_data->handle, &creq); |
| 177 | if (ret || creq.status) { |
| 178 | error_setg(errp, "multifd %u: failed compression, ret %d status %d", |
| 179 | p->id, ret, creq.status); |
| 180 | return -1; |
| 181 | } |
| 182 | if (creq.dst_len < page_size) { |
| 183 | uadk_data->buf_hdr[i] = cpu_to_be32(creq.dst_len); |
| 184 | prepare_next_iov(p, buf, creq.dst_len); |
| 185 | buf += creq.dst_len; |
| 186 | } |
| 187 | } |
| 188 | /* |
| 189 | * Send raw data if no UADK hardware or if compressed out >= page_size. |
| 190 | * We might be better off sending raw data if output is slightly less |
| 191 | * than page_size as well because at the receive end we can skip the |
| 192 | * decompression. But it is tricky to find the right number here. |
| 193 | */ |
| 194 | if (!uadk_data->handle || creq.dst_len >= page_size) { |
| 195 | uadk_data->buf_hdr[i] = cpu_to_be32(page_size); |
| 196 | prepare_next_iov(p, pages->block->host + pages->offset[i], |
| 197 | page_size); |
| 198 | buf += page_size; |
| 199 | } |
| 200 | } |
| 201 | out: |
| 202 | p->flags |= MULTIFD_FLAG_UADK; |
| 203 | multifd_send_fill_packet(p); |
| 204 | return 0; |
| 205 | } |
| 206 | |
| 207 | static int multifd_uadk_recv_setup(MultiFDRecvParams *p, Error **errp) |
| 208 | { |
| 209 | struct wd_data *wd; |
| 210 | uint32_t page_size = multifd_ram_page_size(); |
| 211 | uint32_t page_count = multifd_ram_page_count(); |
| 212 | |
| 213 | wd = multifd_uadk_init_sess(page_count, page_size, false, errp); |
| 214 | if (!wd) { |
| 215 | return -1; |
| 216 | } |
| 217 | p->compress_data = wd; |
| 218 | return 0; |
| 219 | } |
| 220 | |
| 221 | static void multifd_uadk_recv_cleanup(MultiFDRecvParams *p) |
| 222 | { |
| 223 | struct wd_data *wd = p->compress_data; |
| 224 | |
| 225 | multifd_uadk_uninit_sess(wd); |
| 226 | p->compress_data = NULL; |
| 227 | } |
| 228 | |
| 229 | static int multifd_uadk_recv(MultiFDRecvParams *p, Error **errp) |
| 230 | { |
| 231 | struct wd_data *uadk_data = p->compress_data; |
| 232 | uint32_t in_size = p->next_packet_size; |
| 233 | uint32_t flags = p->flags & MULTIFD_FLAG_COMPRESSION_MASK; |
| 234 | uint32_t hdr_len = p->normal_num * sizeof(uint32_t); |
| 235 | uint32_t data_len = 0; |
| 236 | uint32_t page_size = multifd_ram_page_size(); |
| 237 | uint8_t *buf = uadk_data->buf; |
| 238 | int ret = 0; |
| 239 | |
| 240 | if (flags != MULTIFD_FLAG_UADK) { |
| 241 | error_setg(errp, "multifd %u: flags received %x flags expected %x", |
| 242 | p->id, flags, MULTIFD_FLAG_ZLIB); |
| 243 | return -1; |
| 244 | } |
| 245 | |
| 246 | multifd_recv_zero_page_process(p); |
| 247 | if (!p->normal_num) { |
| 248 | if (in_size != 0) { |
| 249 | error_setg(errp, "multifd %u: expected empty packet", p->id); |
| 250 | return -1; |
| 251 | } |
| 252 | return 0; |
| 253 | } |
| 254 | |
| 255 | /* read compressed data lengths */ |
| 256 | if (hdr_len >= in_size) { |
| 257 | error_setg(errp, "multifd %u: header len %"PRIu32 |
| 258 | " >= packet size %"PRIu32, p->id, hdr_len, in_size); |
| 259 | return -1; |
| 260 | } |
| 261 | ret = qio_channel_read_all(p->c, (void *) uadk_data->buf_hdr, |
| 262 | hdr_len, errp); |
| 263 | if (ret != 0) { |
| 264 | return ret; |
| 265 | } |
| 266 | |
| 267 | for (int i = 0; i < p->normal_num; i++) { |
| 268 | uadk_data->buf_hdr[i] = be32_to_cpu(uadk_data->buf_hdr[i]); |
| 269 | if (uadk_data->buf_hdr[i] > page_size) { |
| 270 | error_setg(errp, "multifd %u: page %d compressed len %"PRIu32 |
| 271 | " too large", p->id, i, uadk_data->buf_hdr[i]); |
| 272 | return -1; |
| 273 | } |
| 274 | data_len += uadk_data->buf_hdr[i]; |
| 275 | } |
| 276 | |
| 277 | /* read compressed data */ |
| 278 | if (in_size != hdr_len + data_len) { |
| 279 | error_setg(errp, "multifd %u: packet size %"PRIu32 |
| 280 | " != header %"PRIu32" + data %"PRIu32, |
| 281 | p->id, in_size, hdr_len, data_len); |
| 282 | return -1; |
| 283 | } |
| 284 | ret = qio_channel_read_all(p->c, (void *)buf, data_len, errp); |
| 285 | if (ret != 0) { |
| 286 | return ret; |
| 287 | } |
| 288 | |
| 289 | for (int i = 0; i < p->normal_num; i++) { |
| 290 | struct wd_comp_req creq = { |
| 291 | .op_type = WD_DIR_DECOMPRESS, |
| 292 | .src = buf, |
| 293 | .src_len = uadk_data->buf_hdr[i], |
| 294 | .dst = p->host + p->normal[i], |
| 295 | .dst_len = page_size, |
| 296 | }; |
| 297 | |
| 298 | if (uadk_data->buf_hdr[i] == page_size) { |
| 299 | memcpy(p->host + p->normal[i], buf, page_size); |
| 300 | buf += page_size; |
| 301 | continue; |
| 302 | } |
| 303 | |
| 304 | if (unlikely(!uadk_data->handle)) { |
| 305 | error_setg(errp, "multifd %u: UADK HW not available for decompression", |
| 306 | p->id); |
| 307 | return -1; |
| 308 | } |
| 309 | |
| 310 | ret = wd_do_comp_sync(uadk_data->handle, &creq); |
| 311 | if (ret || creq.status) { |
| 312 | error_setg(errp, "multifd %u: failed decompression, ret %d status %d", |
| 313 | p->id, ret, creq.status); |
| 314 | return -1; |
| 315 | } |
| 316 | if (creq.dst_len != page_size) { |
| 317 | error_setg(errp, "multifd %u: decompressed length error", p->id); |
| 318 | return -1; |
| 319 | } |
| 320 | buf += uadk_data->buf_hdr[i]; |
| 321 | } |
| 322 | |
| 323 | return 0; |
| 324 | } |
| 325 | |
| 326 | static const MultiFDMethods multifd_uadk_ops = { |
| 327 | .send_setup = multifd_uadk_send_setup, |
| 328 | .send_cleanup = multifd_uadk_send_cleanup, |
| 329 | .send_prepare = multifd_uadk_send_prepare, |
| 330 | .recv_setup = multifd_uadk_recv_setup, |
| 331 | .recv_cleanup = multifd_uadk_recv_cleanup, |
| 332 | .recv = multifd_uadk_recv, |
| 333 | }; |
| 334 | |
| 335 | static void multifd_uadk_register(void) |
| 336 | { |
| 337 | multifd_register_ops(MULTIFD_COMPRESSION_UADK, &multifd_uadk_ops); |
| 338 | } |
| 339 | migration_init(multifd_uadk_register); |