| 1 | # |
| 2 | # Migration test main engine |
| 3 | # |
| 4 | # Copyright (c) 2016 Red Hat, Inc. |
| 5 | # |
| 6 | # This library is free software; you can redistribute it and/or |
| 7 | # modify it under the terms of the GNU Lesser General Public |
| 8 | # License as published by the Free Software Foundation; either |
| 9 | # version 2.1 of the License, or (at your option) any later version. |
| 10 | # |
| 11 | # This library is distributed in the hope that it will be useful, |
| 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 14 | # Lesser General Public License for more details. |
| 15 | # |
| 16 | # You should have received a copy of the GNU Lesser General Public |
| 17 | # License along with this library; if not, see <http://www.gnu.org/licenses/>. |
| 18 | # |
| 19 | |
| 20 | |
| 21 | import os |
| 22 | import re |
| 23 | import sys |
| 24 | import time |
| 25 | |
| 26 | from guestperf.progress import Progress, ProgressStats |
| 27 | from guestperf.report import Report, ReportResult |
| 28 | from guestperf.timings import TimingRecord, Timings |
| 29 | |
| 30 | try: |
| 31 | from qemu.machine import QEMUMachine |
| 32 | except ModuleNotFoundError as exc: |
| 33 | print( |
| 34 | f"Module '{exc.name}' not found.\n" |
| 35 | "It should be installed as part of the configure-time " |
| 36 | "virtual environment in $builddir/pyvenv.\n" |
| 37 | "Try re-running this script as:\n" |
| 38 | f"> $builddir/run {' '.join(sys.argv)}", |
| 39 | file=sys.stderr |
| 40 | ) |
| 41 | sys.exit(1) |
| 42 | |
| 43 | # multifd supported compression algorithms |
| 44 | MULTIFD_CMP_ALGS = ("zlib", "zstd", "qpl", "uadk") |
| 45 | |
| 46 | class Engine(object): |
| 47 | |
| 48 | def __init__(self, binary, dst_host, kernel, initrd, transport="tcp", |
| 49 | sleep=15, verbose=False, debug=False): |
| 50 | |
| 51 | self._binary = binary # Path to QEMU binary |
| 52 | self._dst_host = dst_host # Hostname of target host |
| 53 | self._kernel = kernel # Path to kernel image |
| 54 | self._initrd = initrd # Path to stress initrd |
| 55 | self._transport = transport # 'unix' or 'tcp' or 'rdma' |
| 56 | self._sleep = sleep |
| 57 | self._verbose = verbose |
| 58 | self._debug = debug |
| 59 | |
| 60 | if debug: |
| 61 | self._verbose = debug |
| 62 | |
| 63 | def _vcpu_timing(self, pid, tid_list): |
| 64 | records = [] |
| 65 | now = time.time() |
| 66 | |
| 67 | jiffies_per_sec = os.sysconf(os.sysconf_names['SC_CLK_TCK']) |
| 68 | for tid in tid_list: |
| 69 | statfile = "/proc/%d/task/%d/stat" % (pid, tid) |
| 70 | with open(statfile, "r") as fh: |
| 71 | stat = fh.readline() |
| 72 | fields = stat.split(" ") |
| 73 | stime = int(fields[13]) |
| 74 | utime = int(fields[14]) |
| 75 | records.append(TimingRecord(tid, now, 1000 * (stime + utime) / jiffies_per_sec)) |
| 76 | return records |
| 77 | |
| 78 | def _cpu_timing(self, pid): |
| 79 | now = time.time() |
| 80 | |
| 81 | jiffies_per_sec = os.sysconf(os.sysconf_names['SC_CLK_TCK']) |
| 82 | statfile = "/proc/%d/stat" % pid |
| 83 | with open(statfile, "r") as fh: |
| 84 | stat = fh.readline() |
| 85 | fields = stat.split(" ") |
| 86 | stime = int(fields[13]) |
| 87 | utime = int(fields[14]) |
| 88 | return TimingRecord(pid, now, 1000 * (stime + utime) / jiffies_per_sec) |
| 89 | |
| 90 | def _migrate_progress(self, vm): |
| 91 | info = vm.cmd("query-migrate") |
| 92 | |
| 93 | if "ram" not in info: |
| 94 | info["ram"] = {} |
| 95 | |
| 96 | return Progress( |
| 97 | info.get("status", "active"), |
| 98 | ProgressStats( |
| 99 | info["ram"].get("transferred", 0), |
| 100 | info["ram"].get("remaining", 0), |
| 101 | info["ram"].get("total", 0), |
| 102 | info["ram"].get("duplicate", 0), |
| 103 | info["ram"].get("skipped", 0), |
| 104 | info["ram"].get("normal", 0), |
| 105 | info["ram"].get("normal-bytes", 0), |
| 106 | info["ram"].get("dirty-pages-rate", 0), |
| 107 | info["ram"].get("mbps", 0), |
| 108 | info["ram"].get("dirty-sync-count", 0) |
| 109 | ), |
| 110 | time.time(), |
| 111 | info.get("total-time", 0), |
| 112 | info.get("downtime", 0), |
| 113 | info.get("expected-downtime", 0), |
| 114 | info.get("setup-time", 0), |
| 115 | info.get("cpu-throttle-percentage", 0), |
| 116 | info.get("dirty-limit-throttle-time-per-round", 0), |
| 117 | info.get("dirty-limit-ring-full-time", 0), |
| 118 | ) |
| 119 | |
| 120 | def _migrate(self, hardware, scenario, src, |
| 121 | dst, connect_uri, defer_migrate): |
| 122 | src_qemu_time = [] |
| 123 | src_vcpu_time = [] |
| 124 | src_pid = src.get_pid() |
| 125 | |
| 126 | vcpus = src.cmd("query-cpus-fast") |
| 127 | src_threads = [] |
| 128 | for vcpu in vcpus: |
| 129 | src_threads.append(vcpu["thread-id"]) |
| 130 | |
| 131 | # XXX how to get dst timings on remote host ? |
| 132 | |
| 133 | if self._verbose: |
| 134 | print("Sleeping %d seconds for initial guest workload run" % self._sleep) |
| 135 | sleep_secs = self._sleep |
| 136 | while sleep_secs > 1: |
| 137 | src_qemu_time.append(self._cpu_timing(src_pid)) |
| 138 | src_vcpu_time.extend(self._vcpu_timing(src_pid, src_threads)) |
| 139 | time.sleep(1) |
| 140 | sleep_secs -= 1 |
| 141 | |
| 142 | if self._verbose: |
| 143 | print("Starting migration") |
| 144 | if scenario._auto_converge: |
| 145 | resp = src.cmd("migrate-set-capabilities", |
| 146 | capabilities = [ |
| 147 | { "capability": "auto-converge", |
| 148 | "state": True } |
| 149 | ]) |
| 150 | resp = src.cmd("migrate-set-parameters", |
| 151 | cpu_throttle_increment=scenario._auto_converge_step) |
| 152 | |
| 153 | if scenario._post_copy: |
| 154 | resp = src.cmd("migrate-set-capabilities", |
| 155 | capabilities = [ |
| 156 | { "capability": "postcopy-ram", |
| 157 | "state": True } |
| 158 | ]) |
| 159 | resp = dst.cmd("migrate-set-capabilities", |
| 160 | capabilities = [ |
| 161 | { "capability": "postcopy-ram", |
| 162 | "state": True } |
| 163 | ]) |
| 164 | |
| 165 | resp = src.cmd("migrate-set-parameters", |
| 166 | max_bandwidth=scenario._bandwidth * 1024 * 1024) |
| 167 | |
| 168 | resp = src.cmd("migrate-set-parameters", |
| 169 | downtime_limit=scenario._downtime) |
| 170 | |
| 171 | if scenario._compression_mt: |
| 172 | resp = src.cmd("migrate-set-capabilities", |
| 173 | capabilities = [ |
| 174 | { "capability": "compress", |
| 175 | "state": True } |
| 176 | ]) |
| 177 | resp = src.cmd("migrate-set-parameters", |
| 178 | compress_threads=scenario._compression_mt_threads) |
| 179 | resp = dst.cmd("migrate-set-capabilities", |
| 180 | capabilities = [ |
| 181 | { "capability": "compress", |
| 182 | "state": True } |
| 183 | ]) |
| 184 | resp = dst.cmd("migrate-set-parameters", |
| 185 | decompress_threads=scenario._compression_mt_threads) |
| 186 | |
| 187 | if scenario._compression_xbzrle: |
| 188 | resp = src.cmd("migrate-set-capabilities", |
| 189 | capabilities = [ |
| 190 | { "capability": "xbzrle", |
| 191 | "state": True } |
| 192 | ]) |
| 193 | resp = dst.cmd("migrate-set-capabilities", |
| 194 | capabilities = [ |
| 195 | { "capability": "xbzrle", |
| 196 | "state": True } |
| 197 | ]) |
| 198 | resp = src.cmd("migrate-set-parameters", |
| 199 | xbzrle_cache_size=( |
| 200 | hardware._mem * |
| 201 | 1024 * 1024 * 1024 / 100 * |
| 202 | scenario._compression_xbzrle_cache)) |
| 203 | |
| 204 | if scenario._multifd: |
| 205 | if (scenario._multifd_compression and |
| 206 | (scenario._multifd_compression not in MULTIFD_CMP_ALGS)): |
| 207 | raise Exception("unsupported multifd compression " |
| 208 | "algorithm: %s" % |
| 209 | scenario._multifd_compression) |
| 210 | |
| 211 | resp = src.cmd("migrate-set-capabilities", |
| 212 | capabilities = [ |
| 213 | { "capability": "multifd", |
| 214 | "state": True } |
| 215 | ]) |
| 216 | resp = src.cmd("migrate-set-parameters", |
| 217 | multifd_channels=scenario._multifd_channels) |
| 218 | resp = dst.cmd("migrate-set-capabilities", |
| 219 | capabilities = [ |
| 220 | { "capability": "multifd", |
| 221 | "state": True } |
| 222 | ]) |
| 223 | resp = dst.cmd("migrate-set-parameters", |
| 224 | multifd_channels=scenario._multifd_channels) |
| 225 | |
| 226 | if scenario._multifd_compression: |
| 227 | resp = src.cmd("migrate-set-parameters", |
| 228 | multifd_compression=scenario._multifd_compression) |
| 229 | resp = dst.cmd("migrate-set-parameters", |
| 230 | multifd_compression=scenario._multifd_compression) |
| 231 | |
| 232 | if scenario._dirty_limit: |
| 233 | if not hardware._dirty_ring_size: |
| 234 | raise Exception("dirty ring size must be configured when " |
| 235 | "testing dirty limit migration") |
| 236 | |
| 237 | resp = src.cmd("migrate-set-capabilities", |
| 238 | capabilities = [ |
| 239 | { "capability": "dirty-limit", |
| 240 | "state": True } |
| 241 | ]) |
| 242 | resp = src.cmd("migrate-set-parameters", |
| 243 | x_vcpu_dirty_limit_period=scenario._x_vcpu_dirty_limit_period) |
| 244 | resp = src.cmd("migrate-set-parameters", |
| 245 | vcpu_dirty_limit=scenario._vcpu_dirty_limit) |
| 246 | |
| 247 | if defer_migrate: |
| 248 | resp = dst.cmd("migrate-incoming", uri=connect_uri) |
| 249 | resp = src.cmd("migrate", uri=connect_uri) |
| 250 | |
| 251 | post_copy = False |
| 252 | paused = False |
| 253 | |
| 254 | progress_history = [] |
| 255 | |
| 256 | start = time.time() |
| 257 | loop = 0 |
| 258 | while True: |
| 259 | loop = loop + 1 |
| 260 | time.sleep(0.05) |
| 261 | |
| 262 | progress = self._migrate_progress(src) |
| 263 | if (loop % 20) == 0: |
| 264 | src_qemu_time.append(self._cpu_timing(src_pid)) |
| 265 | src_vcpu_time.extend(self._vcpu_timing(src_pid, src_threads)) |
| 266 | |
| 267 | if (len(progress_history) == 0 or |
| 268 | (progress_history[-1]._ram._iterations < |
| 269 | progress._ram._iterations)): |
| 270 | progress_history.append(progress) |
| 271 | |
| 272 | if progress._status in ("completed", "failed", "cancelled"): |
| 273 | if progress._status == "completed" and paused: |
| 274 | dst.cmd("cont") |
| 275 | if progress_history[-1] != progress: |
| 276 | progress_history.append(progress) |
| 277 | |
| 278 | if progress._status == "completed": |
| 279 | if self._verbose: |
| 280 | print("Sleeping %d seconds for final guest workload run" % self._sleep) |
| 281 | sleep_secs = self._sleep |
| 282 | while sleep_secs > 1: |
| 283 | time.sleep(1) |
| 284 | src_qemu_time.append(self._cpu_timing(src_pid)) |
| 285 | src_vcpu_time.extend(self._vcpu_timing(src_pid, src_threads)) |
| 286 | sleep_secs -= 1 |
| 287 | |
| 288 | result = ReportResult() |
| 289 | if progress._status == "completed" and not paused: |
| 290 | result = ReportResult(True) |
| 291 | |
| 292 | return [progress_history, src_qemu_time, src_vcpu_time, result] |
| 293 | |
| 294 | if self._verbose and (loop % 20) == 0: |
| 295 | print("Iter %d: remain %5dMB of %5dMB (total %5dMB @ %5dMb/sec)" % ( |
| 296 | progress._ram._iterations, |
| 297 | progress._ram._remaining_bytes / (1024 * 1024), |
| 298 | progress._ram._total_bytes / (1024 * 1024), |
| 299 | progress._ram._transferred_bytes / (1024 * 1024), |
| 300 | progress._ram._transfer_rate_mbs, |
| 301 | )) |
| 302 | |
| 303 | if progress._ram._iterations > scenario._max_iters: |
| 304 | if self._verbose: |
| 305 | print("No completion after %d iterations over RAM" % scenario._max_iters) |
| 306 | src.cmd("migrate_cancel") |
| 307 | continue |
| 308 | |
| 309 | if time.time() > (start + scenario._max_time): |
| 310 | if self._verbose: |
| 311 | print("No completion after %d seconds" % scenario._max_time) |
| 312 | src.cmd("migrate_cancel") |
| 313 | continue |
| 314 | |
| 315 | if (scenario._post_copy and |
| 316 | progress._ram._iterations >= scenario._post_copy_iters and |
| 317 | not post_copy): |
| 318 | if self._verbose: |
| 319 | print("Switching to post-copy after %d iterations" % scenario._post_copy_iters) |
| 320 | resp = src.cmd("migrate-start-postcopy") |
| 321 | post_copy = True |
| 322 | |
| 323 | if (scenario._pause and |
| 324 | progress._ram._iterations >= scenario._pause_iters and |
| 325 | not paused): |
| 326 | if self._verbose: |
| 327 | print("Pausing VM after %d iterations" % scenario._pause_iters) |
| 328 | resp = src.cmd("stop") |
| 329 | paused = True |
| 330 | |
| 331 | def _is_ppc64le(self): |
| 332 | _, _, _, _, machine = os.uname() |
| 333 | if machine == "ppc64le": |
| 334 | return True |
| 335 | return False |
| 336 | |
| 337 | def _get_guest_console_args(self): |
| 338 | if self._is_ppc64le(): |
| 339 | return "console=hvc0" |
| 340 | else: |
| 341 | return "console=ttyS0" |
| 342 | |
| 343 | def _get_qemu_serial_args(self): |
| 344 | if self._is_ppc64le(): |
| 345 | return ["-chardev", "stdio,id=cdev0", |
| 346 | "-device", "spapr-vty,chardev=cdev0"] |
| 347 | else: |
| 348 | return ["-chardev", "stdio,id=cdev0", |
| 349 | "-device", "isa-serial,chardev=cdev0"] |
| 350 | |
| 351 | def _get_common_args(self, hardware, tunnelled=False): |
| 352 | args = [ |
| 353 | "noapic", |
| 354 | "edd=off", |
| 355 | "printk.time=1", |
| 356 | "noreplace-smp", |
| 357 | "cgroup_disable=memory", |
| 358 | "pci=noearly", |
| 359 | ] |
| 360 | |
| 361 | args.append(self._get_guest_console_args()) |
| 362 | |
| 363 | if self._debug: |
| 364 | args.append("debug") |
| 365 | else: |
| 366 | args.append("quiet") |
| 367 | |
| 368 | args.append("ramsize=%s" % hardware._mem) |
| 369 | |
| 370 | cmdline = " ".join(args) |
| 371 | if tunnelled: |
| 372 | cmdline = "'" + cmdline + "'" |
| 373 | |
| 374 | argv = [ |
| 375 | "-cpu", "host", |
| 376 | "-kernel", self._kernel, |
| 377 | "-initrd", self._initrd, |
| 378 | "-append", cmdline, |
| 379 | "-m", str((hardware._mem * 1024) + 512), |
| 380 | "-smp", str(hardware._cpus), |
| 381 | ] |
| 382 | if hardware._dirty_ring_size: |
| 383 | argv.extend(["-accel", "kvm,dirty-ring-size=%s" % |
| 384 | hardware._dirty_ring_size]) |
| 385 | else: |
| 386 | argv.extend(["-accel", "kvm"]) |
| 387 | |
| 388 | argv.extend(self._get_qemu_serial_args()) |
| 389 | |
| 390 | if self._debug: |
| 391 | argv.extend(["-machine", "graphics=off"]) |
| 392 | |
| 393 | if hardware._prealloc_pages: |
| 394 | argv_source += ["-mem-path", "/dev/shm", |
| 395 | "-mem-prealloc"] |
| 396 | if hardware._locked_pages: |
| 397 | argv_source += ["-overcommit", "mem-lock=on"] |
| 398 | if hardware._huge_pages: |
| 399 | pass |
| 400 | |
| 401 | return argv |
| 402 | |
| 403 | def _get_src_args(self, hardware): |
| 404 | return self._get_common_args(hardware) |
| 405 | |
| 406 | def _get_dst_args(self, hardware, uri, defer_migrate): |
| 407 | tunnelled = False |
| 408 | if self._dst_host != "localhost": |
| 409 | tunnelled = True |
| 410 | argv = self._get_common_args(hardware, tunnelled) |
| 411 | |
| 412 | if defer_migrate: |
| 413 | return argv + ["-incoming", "defer"] |
| 414 | return argv + ["-incoming", uri] |
| 415 | |
| 416 | @staticmethod |
| 417 | def _get_common_wrapper(cpu_bind, mem_bind): |
| 418 | wrapper = [] |
| 419 | if len(cpu_bind) > 0 or len(mem_bind) > 0: |
| 420 | wrapper.append("numactl") |
| 421 | if cpu_bind: |
| 422 | wrapper.append("--physcpubind=%s" % ",".join(cpu_bind)) |
| 423 | if mem_bind: |
| 424 | wrapper.append("--membind=%s" % ",".join(mem_bind)) |
| 425 | |
| 426 | return wrapper |
| 427 | |
| 428 | def _get_src_wrapper(self, hardware): |
| 429 | return self._get_common_wrapper(hardware._src_cpu_bind, hardware._src_mem_bind) |
| 430 | |
| 431 | def _get_dst_wrapper(self, hardware): |
| 432 | wrapper = self._get_common_wrapper(hardware._dst_cpu_bind, hardware._dst_mem_bind) |
| 433 | if self._dst_host != "localhost": |
| 434 | return ["ssh", |
| 435 | "-R", "9001:localhost:9001", |
| 436 | self._dst_host] + wrapper |
| 437 | else: |
| 438 | return wrapper |
| 439 | |
| 440 | def _get_timings(self, vm): |
| 441 | log = vm.get_log() |
| 442 | if not log: |
| 443 | return [] |
| 444 | if self._debug: |
| 445 | print(log) |
| 446 | |
| 447 | regex = r"[^\s]+\s\((\d+)\):\sINFO:\s(\d+)ms\scopied\s\d+\sGB\sin\s(\d+)ms" |
| 448 | matcher = re.compile(regex) |
| 449 | records = [] |
| 450 | for line in log.split("\n"): |
| 451 | match = matcher.match(line) |
| 452 | if match: |
| 453 | records.append(TimingRecord(int(match.group(1)), |
| 454 | int(match.group(2)) / 1000.0, |
| 455 | int(match.group(3)))) |
| 456 | return records |
| 457 | |
| 458 | def run(self, hardware, scenario, result_dir=os.getcwd()): |
| 459 | abs_result_dir = os.path.join(result_dir, scenario._name) |
| 460 | defer_migrate = False |
| 461 | |
| 462 | if self._transport == "tcp": |
| 463 | uri = "tcp:%s:9000" % self._dst_host |
| 464 | elif self._transport == "rdma": |
| 465 | uri = "rdma:%s:9000" % self._dst_host |
| 466 | elif self._transport == "unix": |
| 467 | if self._dst_host != "localhost": |
| 468 | raise Exception("Running use unix migration transport for non-local host") |
| 469 | uri = "unix:/var/tmp/qemu-migrate-%d.migrate" % os.getpid() |
| 470 | try: |
| 471 | os.remove(uri[5:]) |
| 472 | os.remove(monaddr) |
| 473 | except: |
| 474 | pass |
| 475 | |
| 476 | if scenario._multifd: |
| 477 | defer_migrate = True |
| 478 | |
| 479 | if self._dst_host != "localhost": |
| 480 | dstmonaddr = ("localhost", 9001) |
| 481 | else: |
| 482 | dstmonaddr = "/var/tmp/qemu-dst-%d-monitor.sock" % os.getpid() |
| 483 | srcmonaddr = "/var/tmp/qemu-src-%d-monitor.sock" % os.getpid() |
| 484 | |
| 485 | src = QEMUMachine(self._binary, |
| 486 | args=self._get_src_args(hardware), |
| 487 | wrapper=self._get_src_wrapper(hardware), |
| 488 | name="qemu-src-%d" % os.getpid(), |
| 489 | monitor_address=srcmonaddr) |
| 490 | |
| 491 | dst = QEMUMachine(self._binary, |
| 492 | args=self._get_dst_args(hardware, uri, defer_migrate), |
| 493 | wrapper=self._get_dst_wrapper(hardware), |
| 494 | name="qemu-dst-%d" % os.getpid(), |
| 495 | monitor_address=dstmonaddr) |
| 496 | |
| 497 | try: |
| 498 | src.launch() |
| 499 | dst.launch() |
| 500 | |
| 501 | ret = self._migrate(hardware, scenario, src, |
| 502 | dst, uri, defer_migrate) |
| 503 | progress_history = ret[0] |
| 504 | qemu_timings = ret[1] |
| 505 | vcpu_timings = ret[2] |
| 506 | result = ret[3] |
| 507 | if uri[0:5] == "unix:" and os.path.exists(uri[5:]): |
| 508 | os.remove(uri[5:]) |
| 509 | |
| 510 | if os.path.exists(srcmonaddr): |
| 511 | os.remove(srcmonaddr) |
| 512 | |
| 513 | if self._dst_host == "localhost" and os.path.exists(dstmonaddr): |
| 514 | os.remove(dstmonaddr) |
| 515 | |
| 516 | if self._verbose: |
| 517 | print("Finished migration") |
| 518 | |
| 519 | src.shutdown() |
| 520 | dst.shutdown() |
| 521 | |
| 522 | return Report(hardware, scenario, progress_history, |
| 523 | Timings(self._get_timings(src) + self._get_timings(dst)), |
| 524 | Timings(qemu_timings), |
| 525 | Timings(vcpu_timings), |
| 526 | result, |
| 527 | self._binary, self._dst_host, self._kernel, |
| 528 | self._initrd, self._transport, self._sleep) |
| 529 | except Exception as e: |
| 530 | if self._debug: |
| 531 | print("Failed: %s" % str(e)) |
| 532 | try: |
| 533 | src.shutdown() |
| 534 | except: |
| 535 | pass |
| 536 | try: |
| 537 | dst.shutdown() |
| 538 | except: |
| 539 | pass |
| 540 | |
| 541 | if self._debug: |
| 542 | print(src.get_log()) |
| 543 | print(dst.get_log()) |
| 544 | raise |
| 545 |