| 1 | # TestEnv class to manage test environment variables. |
| 2 | # |
| 3 | # Copyright (c) 2020-2021 Virtuozzo International GmbH |
| 4 | # |
| 5 | # This program is free software; you can redistribute it and/or modify |
| 6 | # it under the terms of the GNU General Public License as published by |
| 7 | # the Free Software Foundation; either version 2 of the License, or |
| 8 | # (at your option) any later version. |
| 9 | # |
| 10 | # This program is distributed in the hope that it will be useful, |
| 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 | # GNU General Public License for more details. |
| 14 | # |
| 15 | # You should have received a copy of the GNU General Public License |
| 16 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 17 | # |
| 18 | |
| 19 | import os |
| 20 | import sys |
| 21 | import tempfile |
| 22 | from pathlib import Path |
| 23 | import shlex |
| 24 | import shutil |
| 25 | import collections |
| 26 | import contextlib |
| 27 | import random |
| 28 | import subprocess |
| 29 | import glob |
| 30 | from typing import List, Dict, Any, Optional |
| 31 | |
| 32 | |
| 33 | DEF_GDB_OPTIONS = 'localhost:12345' |
| 34 | |
| 35 | def isxfile(path: str) -> bool: |
| 36 | return os.path.isfile(path) and os.access(path, os.X_OK) |
| 37 | |
| 38 | |
| 39 | def get_default_machine(qemu_prog: str) -> str: |
| 40 | outp = subprocess.run([qemu_prog, '-machine', 'help'], check=True, |
| 41 | universal_newlines=True, |
| 42 | stdout=subprocess.PIPE).stdout |
| 43 | |
| 44 | machines = outp.split('\n') |
| 45 | try: |
| 46 | default_machine = next(m for m in machines if ' (default)' in m) |
| 47 | except StopIteration: |
| 48 | return '' |
| 49 | default_machine = default_machine.split(' ', 1)[0] |
| 50 | |
| 51 | alias_suf = ' (alias of {})'.format(default_machine) |
| 52 | alias = next((m for m in machines if m.endswith(alias_suf)), None) |
| 53 | if alias is not None: |
| 54 | default_machine = alias.split(' ', 1)[0] |
| 55 | |
| 56 | return default_machine |
| 57 | |
| 58 | |
| 59 | class TestEnv(contextlib.AbstractContextManager['TestEnv']): |
| 60 | """ |
| 61 | Manage system environment for running tests |
| 62 | |
| 63 | The following variables are supported/provided. They are represented by |
| 64 | lower-cased TestEnv attributes. |
| 65 | """ |
| 66 | |
| 67 | # We store environment variables as instance attributes, and there are a |
| 68 | # lot of them. Silence pylint: |
| 69 | # pylint: disable=too-many-instance-attributes |
| 70 | |
| 71 | env_variables = ['PYTHONPATH', 'TEST_DIR', 'SOCK_DIR', 'SAMPLE_IMG_DIR', |
| 72 | 'PYTHON', 'QEMU_PROG', 'QEMU_IMG_PROG', |
| 73 | 'QEMU_IO_PROG', 'QEMU_NBD_PROG', 'QSD_PROG', |
| 74 | 'QEMU_OPTIONS', 'QEMU_IMG_OPTIONS', |
| 75 | 'QEMU_IO_OPTIONS', 'QEMU_IO_OPTIONS_NO_FMT', |
| 76 | 'QEMU_NBD_OPTIONS', 'IMGOPTS', 'IMGFMT', 'IMGPROTO', |
| 77 | 'AIOMODE', 'CACHEMODE', 'VALGRIND_QEMU', |
| 78 | 'CACHEMODE_IS_DEFAULT', 'IMGFMT_GENERIC', 'IMGOPTSSYNTAX', |
| 79 | 'IMGKEYSECRET', 'QEMU_DEFAULT_MACHINE', 'MALLOC_PERTURB_', |
| 80 | 'GDB_OPTIONS', 'PRINT_QEMU'] |
| 81 | |
| 82 | def prepare_subprocess(self, args: List[str]) -> Dict[str, str]: |
| 83 | if self.debug: |
| 84 | args.append('-d') |
| 85 | |
| 86 | with open(args[0], encoding="utf-8") as f: |
| 87 | try: |
| 88 | if f.readline().rstrip() == '#!/usr/bin/env python3': |
| 89 | args.insert(0, self.python) |
| 90 | except UnicodeDecodeError: # binary test? for future. |
| 91 | pass |
| 92 | |
| 93 | os_env = os.environ.copy() |
| 94 | os_env.update(self.get_env()) |
| 95 | return os_env |
| 96 | |
| 97 | def get_env(self) -> Dict[str, str]: |
| 98 | env = {} |
| 99 | for v in self.env_variables: |
| 100 | val = getattr(self, v.lower(), None) |
| 101 | if val is not None: |
| 102 | env[v] = val |
| 103 | |
| 104 | return env |
| 105 | |
| 106 | def init_directories(self) -> None: |
| 107 | """Init directory variables: |
| 108 | PYTHONPATH |
| 109 | TEST_DIR |
| 110 | SOCK_DIR |
| 111 | SAMPLE_IMG_DIR |
| 112 | """ |
| 113 | |
| 114 | # Path where qemu goodies live in this source tree. |
| 115 | qemu_srctree_path = Path(__file__, '../../../python').resolve() |
| 116 | |
| 117 | self.pythonpath = os.pathsep.join(filter(None, ( |
| 118 | self.source_iotests, |
| 119 | str(qemu_srctree_path), |
| 120 | os.getenv('PYTHONPATH'), |
| 121 | ))) |
| 122 | |
| 123 | self.test_dir = os.getenv('TEST_DIR', |
| 124 | os.path.join(os.getcwd(), 'scratch')) |
| 125 | Path(self.test_dir).mkdir(parents=True, exist_ok=True) |
| 126 | |
| 127 | try: |
| 128 | self.sock_dir = os.environ['SOCK_DIR'] |
| 129 | self.tmp_sock_dir = False |
| 130 | Path(self.sock_dir).mkdir(parents=True, exist_ok=True) |
| 131 | except KeyError: |
| 132 | self.sock_dir = tempfile.mkdtemp(prefix="qemu-iotests-") |
| 133 | self.tmp_sock_dir = True |
| 134 | |
| 135 | self.sample_img_dir = os.getenv('SAMPLE_IMG_DIR', |
| 136 | os.path.join(self.source_iotests, |
| 137 | 'sample_images')) |
| 138 | |
| 139 | def init_binaries(self) -> None: |
| 140 | """Init binary path variables: |
| 141 | PYTHON (for bash tests) |
| 142 | QEMU_PROG, QEMU_IMG_PROG, QEMU_IO_PROG, QEMU_NBD_PROG, QSD_PROG |
| 143 | """ |
| 144 | self.python = str(Path(sys.executable).absolute()) |
| 145 | |
| 146 | # QEMU configure-time venv python executable |
| 147 | venv_python = Path( |
| 148 | os.path.join(self.build_root, "pyvenv", "bin", "python3") |
| 149 | ).absolute() |
| 150 | |
| 151 | if self.python != str(venv_python): |
| 152 | runpath = os.path.join(self.build_root, "run") |
| 153 | cmd = ' '.join(shlex.quote(x) for x in sys.argv) |
| 154 | print( |
| 155 | "\n\033[93m\033[1mWARNING\033[0m: " |
| 156 | "iotests is being run from outside of the configure-time " |
| 157 | "python virtual environment\n\n" |
| 158 | f"current python: {self.python}\n" |
| 159 | f"pyvenv python: {venv_python}\n\n" |
| 160 | "Individual python tests will be executed inside the pyvenv,\n" |
| 161 | "but the test runner will continue to run outside.\n\n" |
| 162 | "\033[1mPlease use the meson run script:\033[0m\n" |
| 163 | f"\t{runpath} {cmd}\n", |
| 164 | file=sys.stderr |
| 165 | ) |
| 166 | self.python = str(venv_python) |
| 167 | |
| 168 | def root(*names: str) -> str: |
| 169 | return os.path.join(self.build_root, *names) |
| 170 | |
| 171 | arch = os.uname().machine |
| 172 | if 'ppc64' in arch: |
| 173 | arch = 'ppc64' |
| 174 | |
| 175 | self.qemu_prog = os.getenv('QEMU_PROG', root(f'qemu-system-{arch}')) |
| 176 | if not os.path.exists(self.qemu_prog): |
| 177 | pattern = root('qemu-system-*') |
| 178 | try: |
| 179 | progs = sorted(glob.iglob(pattern)) |
| 180 | self.qemu_prog = next(p for p in progs if isxfile(p)) |
| 181 | except StopIteration: |
| 182 | sys.exit("Not found any Qemu executable binary by pattern " |
| 183 | f"'{pattern}'") |
| 184 | |
| 185 | self.qemu_img_prog = os.getenv('QEMU_IMG_PROG', root('qemu-img')) |
| 186 | self.qemu_io_prog = os.getenv('QEMU_IO_PROG', root('qemu-io')) |
| 187 | self.qemu_nbd_prog = os.getenv('QEMU_NBD_PROG', root('qemu-nbd')) |
| 188 | self.qsd_prog = os.getenv('QSD_PROG', root('storage-daemon', |
| 189 | 'qemu-storage-daemon')) |
| 190 | |
| 191 | for b in [self.qemu_img_prog, self.qemu_io_prog, self.qemu_nbd_prog, |
| 192 | self.qemu_prog, self.qsd_prog]: |
| 193 | if not os.path.exists(b): |
| 194 | sys.exit('No such file: ' + b) |
| 195 | if not isxfile(b): |
| 196 | sys.exit('Not executable: ' + b) |
| 197 | |
| 198 | def __init__(self, source_dir: str, build_dir: str, |
| 199 | imgfmt: str, imgproto: str, aiomode: str, |
| 200 | cachemode: Optional[str] = None, |
| 201 | imgopts: Optional[str] = None, |
| 202 | misalign: bool = False, |
| 203 | debug: bool = False, |
| 204 | valgrind: bool = False, |
| 205 | gdb: bool = False, |
| 206 | qprint: bool = False, |
| 207 | dry_run: bool = False) -> None: |
| 208 | self.imgfmt = imgfmt |
| 209 | self.imgproto = imgproto |
| 210 | self.aiomode = aiomode |
| 211 | self.imgopts = imgopts |
| 212 | self.misalign = misalign |
| 213 | self.debug = debug |
| 214 | |
| 215 | if qprint: |
| 216 | self.print_qemu = 'y' |
| 217 | |
| 218 | if gdb: |
| 219 | self.gdb_options = os.getenv('GDB_OPTIONS', DEF_GDB_OPTIONS) |
| 220 | if not self.gdb_options: |
| 221 | # cover the case 'export GDB_OPTIONS=' |
| 222 | self.gdb_options = DEF_GDB_OPTIONS |
| 223 | elif 'GDB_OPTIONS' in os.environ: |
| 224 | # to not propagate it in prepare_subprocess() |
| 225 | del os.environ['GDB_OPTIONS'] |
| 226 | |
| 227 | if valgrind: |
| 228 | self.valgrind_qemu = 'y' |
| 229 | |
| 230 | if cachemode is None: |
| 231 | self.cachemode_is_default = 'true' |
| 232 | self.cachemode = 'writeback' |
| 233 | else: |
| 234 | self.cachemode_is_default = 'false' |
| 235 | self.cachemode = cachemode |
| 236 | |
| 237 | # Initialize generic paths: build_root, build_iotests, source_iotests, |
| 238 | # which are needed to initialize some environment variables. They are |
| 239 | # used by init_*() functions as well. |
| 240 | |
| 241 | self.source_iotests = source_dir |
| 242 | self.build_iotests = build_dir |
| 243 | |
| 244 | self.build_root = Path(self.build_iotests).parent.parent |
| 245 | |
| 246 | self.init_directories() |
| 247 | |
| 248 | if dry_run: |
| 249 | return |
| 250 | |
| 251 | self.init_binaries() |
| 252 | |
| 253 | self.malloc_perturb_ = os.getenv('MALLOC_PERTURB_', |
| 254 | str(random.randrange(1, 255))) |
| 255 | |
| 256 | # QEMU_OPTIONS |
| 257 | self.qemu_options = '-nodefaults -display none -accel qtest' |
| 258 | machine_map = ( |
| 259 | ('arm', 'virt'), |
| 260 | ('aarch64', 'virt'), |
| 261 | ('avr', 'mega2560'), |
| 262 | ('hexagon', 'virt'), |
| 263 | ('m68k', 'virt'), |
| 264 | ('or1k', 'virt'), |
| 265 | ('riscv32', 'virt'), |
| 266 | ('riscv64', 'virt'), |
| 267 | ('rx', 'gdbsim-r5f562n8'), |
| 268 | ('sh4', 'r2d'), |
| 269 | ('sh4eb', 'r2d'), |
| 270 | ('tricore', 'tricore_testboard') |
| 271 | ) |
| 272 | for suffix, machine in machine_map: |
| 273 | if self.qemu_prog.endswith(f'qemu-system-{suffix}'): |
| 274 | self.qemu_options += f' -machine {machine}' |
| 275 | |
| 276 | # QEMU_DEFAULT_MACHINE |
| 277 | self.qemu_default_machine = get_default_machine(self.qemu_prog) |
| 278 | |
| 279 | self.qemu_img_options = os.getenv('QEMU_IMG_OPTIONS') |
| 280 | self.qemu_nbd_options = os.getenv('QEMU_NBD_OPTIONS') |
| 281 | |
| 282 | is_generic = self.imgfmt not in ['bochs', 'cloop', 'dmg', 'vvfat'] |
| 283 | self.imgfmt_generic = 'true' if is_generic else 'false' |
| 284 | |
| 285 | self.qemu_io_options = f'--cache {self.cachemode} --aio {self.aiomode}' |
| 286 | if self.misalign: |
| 287 | self.qemu_io_options += ' --misalign' |
| 288 | |
| 289 | self.qemu_io_options_no_fmt = self.qemu_io_options |
| 290 | |
| 291 | if self.imgfmt == 'luks': |
| 292 | self.imgoptssyntax = 'true' |
| 293 | self.imgkeysecret = '123456' |
| 294 | if not self.imgopts: |
| 295 | self.imgopts = 'iter-time=10' |
| 296 | elif 'iter-time=' not in self.imgopts: |
| 297 | self.imgopts += ',iter-time=10' |
| 298 | else: |
| 299 | self.imgoptssyntax = 'false' |
| 300 | self.qemu_io_options += ' -f ' + self.imgfmt |
| 301 | |
| 302 | if self.imgfmt == 'vmdk': |
| 303 | if not self.imgopts: |
| 304 | self.imgopts = 'zeroed_grain=on' |
| 305 | elif 'zeroed_grain=' not in self.imgopts: |
| 306 | self.imgopts += ',zeroed_grain=on' |
| 307 | |
| 308 | def close(self) -> None: |
| 309 | if self.tmp_sock_dir: |
| 310 | shutil.rmtree(self.sock_dir) |
| 311 | |
| 312 | def __enter__(self) -> 'TestEnv': |
| 313 | return self |
| 314 | |
| 315 | def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: |
| 316 | self.close() |
| 317 | |
| 318 | def print_env(self, prefix: str = '') -> None: |
| 319 | template = """\ |
| 320 | {prefix}QEMU -- "{QEMU_PROG}" {QEMU_OPTIONS} |
| 321 | {prefix}QEMU_IMG -- "{QEMU_IMG_PROG}" {QEMU_IMG_OPTIONS} |
| 322 | {prefix}QEMU_IO -- "{QEMU_IO_PROG}" {QEMU_IO_OPTIONS} |
| 323 | {prefix}QEMU_NBD -- "{QEMU_NBD_PROG}" {QEMU_NBD_OPTIONS} |
| 324 | {prefix}IMGFMT -- {IMGFMT}{imgopts} |
| 325 | {prefix}IMGPROTO -- {IMGPROTO} |
| 326 | {prefix}PLATFORM -- {platform} |
| 327 | {prefix}TEST_DIR -- {TEST_DIR} |
| 328 | {prefix}SOCK_DIR -- {SOCK_DIR} |
| 329 | {prefix}GDB_OPTIONS -- {GDB_OPTIONS} |
| 330 | {prefix}VALGRIND_QEMU -- {VALGRIND_QEMU} |
| 331 | {prefix}PRINT_QEMU_OUTPUT -- {PRINT_QEMU} |
| 332 | {prefix}""" |
| 333 | |
| 334 | args = collections.defaultdict(str, self.get_env()) |
| 335 | |
| 336 | if 'IMGOPTS' in args: |
| 337 | args['imgopts'] = f" ({args['IMGOPTS']})" |
| 338 | |
| 339 | u = os.uname() |
| 340 | args['platform'] = f'{u.sysname}/{u.machine} {u.nodename} {u.release}' |
| 341 | args['prefix'] = prefix |
| 342 | print(template.format_map(args)) |