| 1 | """ |
| 2 | mkvenv - QEMU pyvenv bootstrapping utility |
| 3 | |
| 4 | usage: mkvenv [-h] command ... |
| 5 | |
| 6 | QEMU pyvenv bootstrapping utility |
| 7 | |
| 8 | options: |
| 9 | -h, --help show this help message and exit |
| 10 | |
| 11 | Commands: |
| 12 | command Description |
| 13 | create create a venv |
| 14 | post_init |
| 15 | post-venv initialization |
| 16 | ensuregroup |
| 17 | Ensure that the specified package group is installed. |
| 18 | |
| 19 | -------------------------------------------------- |
| 20 | |
| 21 | usage: mkvenv create [-h] target |
| 22 | |
| 23 | positional arguments: |
| 24 | target Target directory to install virtual environment into. |
| 25 | |
| 26 | options: |
| 27 | -h, --help show this help message and exit |
| 28 | |
| 29 | -------------------------------------------------- |
| 30 | |
| 31 | usage: mkvenv post_init [-h] |
| 32 | |
| 33 | options: |
| 34 | -h, --help show this help message and exit |
| 35 | |
| 36 | -------------------------------------------------- |
| 37 | |
| 38 | usage: mkvenv ensuregroup [-h] [--online] [--dir DIR] file group... |
| 39 | |
| 40 | positional arguments: |
| 41 | file pointer to a TOML file |
| 42 | group section name in the TOML file |
| 43 | |
| 44 | options: |
| 45 | -h, --help show this help message and exit |
| 46 | --online Install packages from PyPI, if necessary. |
| 47 | --dir DIR Path to vendored packages where we may install from. |
| 48 | |
| 49 | """ |
| 50 | |
| 51 | # Copyright (C) 2022-2023 Red Hat, Inc. |
| 52 | # |
| 53 | # Authors: |
| 54 | # John Snow <jsnow@redhat.com> |
| 55 | # Paolo Bonzini <pbonzini@redhat.com> |
| 56 | # |
| 57 | # This work is licensed under the terms of the GNU GPL, version 2 or |
| 58 | # later. See the COPYING file in the top-level directory. |
| 59 | |
| 60 | import argparse |
| 61 | from importlib.metadata import ( |
| 62 | Distribution, |
| 63 | EntryPoint, |
| 64 | PackageNotFoundError, |
| 65 | distribution, |
| 66 | version, |
| 67 | ) |
| 68 | from importlib.util import find_spec |
| 69 | import logging |
| 70 | import os |
| 71 | from pathlib import Path |
| 72 | import re |
| 73 | import shutil |
| 74 | import site |
| 75 | import subprocess |
| 76 | import sys |
| 77 | import sysconfig |
| 78 | from types import SimpleNamespace |
| 79 | from typing import ( |
| 80 | Any, |
| 81 | Dict, |
| 82 | Iterator, |
| 83 | Optional, |
| 84 | Sequence, |
| 85 | Tuple, |
| 86 | Union, |
| 87 | cast, |
| 88 | ) |
| 89 | import venv |
| 90 | |
| 91 | |
| 92 | # Try to load distlib, with a fallback to pip's vendored version. |
| 93 | # HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail |
| 94 | # outside the venv or before a potential call to ensurepip in checkpip(). |
| 95 | _import_ok = True |
| 96 | try: |
| 97 | import distlib.scripts |
| 98 | except ImportError: |
| 99 | try: |
| 100 | # Reach into pip's cookie jar. pylint and flake8 don't understand |
| 101 | # that these imports will be used via distlib.xxx. |
| 102 | from pip._vendor import distlib |
| 103 | import pip._vendor.distlib.scripts # noqa, pylint: disable=unused-import |
| 104 | except ImportError: |
| 105 | _import_ok = False |
| 106 | |
| 107 | HAVE_DISTLIB = _import_ok |
| 108 | |
| 109 | # pip 25.2 does not vendor distlib.version, but it uses vendored |
| 110 | # packaging.version |
| 111 | _import_ok = True |
| 112 | try: |
| 113 | import distlib.version # pylint: disable=ungrouped-imports |
| 114 | except ImportError: |
| 115 | try: |
| 116 | # pylint: disable=unused-import,ungrouped-imports |
| 117 | import pip._vendor.distlib.version # noqa |
| 118 | except ImportError: |
| 119 | _import_ok = False |
| 120 | |
| 121 | HAVE_DISTLIB_VERSION = _import_ok |
| 122 | |
| 123 | _import_ok = True |
| 124 | try: |
| 125 | # Do not bother importing non-vendored packaging, because it is not |
| 126 | # in stdlib. |
| 127 | from pip._vendor import packaging |
| 128 | # pylint: disable=unused-import |
| 129 | import pip._vendor.packaging.requirements # noqa |
| 130 | import pip._vendor.packaging.version # noqa |
| 131 | except ImportError: |
| 132 | _import_ok = False |
| 133 | |
| 134 | HAVE_PACKAGING_VERSION = _import_ok |
| 135 | |
| 136 | |
| 137 | # Try to load tomllib, with a fallback to tomli. |
| 138 | # HAVE_TOMLLIB is checked below, just-in-time, so that mkvenv does not fail |
| 139 | # outside the venv or before a potential call to ensurepip in checkpip(). |
| 140 | _import_ok = True |
| 141 | try: |
| 142 | import tomllib |
| 143 | except ImportError: |
| 144 | try: |
| 145 | import tomli as tomllib |
| 146 | except ImportError: |
| 147 | _import_ok = False |
| 148 | |
| 149 | HAVE_TOMLLIB = _import_ok |
| 150 | |
| 151 | # Do not add any mandatory dependencies from outside the stdlib: |
| 152 | # This script *must* be usable standalone! |
| 153 | |
| 154 | DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] |
| 155 | logger = logging.getLogger("mkvenv") |
| 156 | |
| 157 | |
| 158 | def inside_a_venv() -> bool: |
| 159 | """Returns True if it is executed inside of a virtual environment.""" |
| 160 | return sys.prefix != sys.base_prefix |
| 161 | |
| 162 | |
| 163 | class Ouch(RuntimeError): |
| 164 | """An Exception class we can't confuse with a builtin.""" |
| 165 | |
| 166 | |
| 167 | class Matcher: |
| 168 | """Compatibility appliance for version/requirement string parsing.""" |
| 169 | def __init__(self, name_and_constraint: str): |
| 170 | """Create a matcher from a requirement-like string.""" |
| 171 | if HAVE_DISTLIB_VERSION: |
| 172 | self._m = distlib.version.LegacyMatcher(name_and_constraint) |
| 173 | elif HAVE_PACKAGING_VERSION: |
| 174 | self._m = packaging.requirements.Requirement(name_and_constraint) |
| 175 | else: |
| 176 | raise Ouch("found neither distlib.version nor packaging.version") |
| 177 | self.name = self._m.name |
| 178 | |
| 179 | def match(self, version_str: str) -> bool: |
| 180 | """Return True if `version` satisfies the stored constraint.""" |
| 181 | if HAVE_DISTLIB_VERSION: |
| 182 | return cast( |
| 183 | bool, |
| 184 | self._m.match(distlib.version.LegacyVersion(version_str)) |
| 185 | ) |
| 186 | |
| 187 | assert HAVE_PACKAGING_VERSION |
| 188 | return cast( |
| 189 | bool, |
| 190 | self._m.specifier.contains( |
| 191 | packaging.version.Version(version_str), prereleases=True |
| 192 | ) |
| 193 | ) |
| 194 | |
| 195 | def __str__(self) -> str: |
| 196 | """String representation delegated to the backend.""" |
| 197 | return str(self._m) |
| 198 | |
| 199 | def __repr__(self) -> str: |
| 200 | """Stable debug representation delegated to the backend.""" |
| 201 | return repr(self._m) |
| 202 | |
| 203 | |
| 204 | class QemuEnvBuilder(venv.EnvBuilder): |
| 205 | """ |
| 206 | An extension of venv.EnvBuilder for building QEMU's configure-time venv. |
| 207 | |
| 208 | The primary difference is that it emulates a "nested" virtual |
| 209 | environment when invoked from inside of an existing virtual |
| 210 | environment by including packages from the parent. Also, |
| 211 | "ensurepip" is replaced if possible with just recreating pip's |
| 212 | console_scripts inside the virtual environment. |
| 213 | |
| 214 | Parameters for base class init: |
| 215 | - system_site_packages: bool = False |
| 216 | - clear: bool = False |
| 217 | - symlinks: bool = False |
| 218 | - upgrade: bool = False |
| 219 | - with_pip: bool = False |
| 220 | - prompt: Optional[str] = None |
| 221 | - upgrade_deps: bool = False (Since 3.9) |
| 222 | """ |
| 223 | |
| 224 | def __init__(self, *args: Any, **kwargs: Any) -> None: |
| 225 | logger.debug("QemuEnvBuilder.__init__(...)") |
| 226 | |
| 227 | # For nested venv emulation: |
| 228 | self.use_parent_packages = False |
| 229 | if inside_a_venv(): |
| 230 | # Include parent packages only if we're in a venv and |
| 231 | # system_site_packages was True. |
| 232 | self.use_parent_packages = kwargs.pop( |
| 233 | "system_site_packages", False |
| 234 | ) |
| 235 | # Include system_site_packages only when the parent, |
| 236 | # The venv we are currently in, also does so. |
| 237 | kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES |
| 238 | |
| 239 | # ensurepip is slow: venv creation can be very fast for cases where |
| 240 | # we allow the use of system_site_packages. Therefore, ensurepip is |
| 241 | # replaced with our own script generation once the virtual environment |
| 242 | # is setup. |
| 243 | self.want_pip = kwargs.get("with_pip", False) |
| 244 | if self.want_pip: |
| 245 | if ( |
| 246 | kwargs.get("system_site_packages", False) |
| 247 | and not need_ensurepip() |
| 248 | ): |
| 249 | kwargs["with_pip"] = False |
| 250 | else: |
| 251 | check_ensurepip() |
| 252 | |
| 253 | super().__init__(*args, **kwargs) |
| 254 | |
| 255 | # Make the context available post-creation: |
| 256 | self._context: Optional[SimpleNamespace] = None |
| 257 | |
| 258 | def get_parent_libpath(self) -> Optional[str]: |
| 259 | """Return the libpath of the parent venv, if applicable.""" |
| 260 | if self.use_parent_packages: |
| 261 | return sysconfig.get_path("purelib") |
| 262 | return None |
| 263 | |
| 264 | @staticmethod |
| 265 | def compute_venv_libpath(context: SimpleNamespace) -> str: |
| 266 | """ |
| 267 | Compatibility wrapper for context.lib_path for Python < 3.12 |
| 268 | """ |
| 269 | # Python 3.12+, not strictly necessary because it's documented |
| 270 | # to be the same as 3.10 code below: |
| 271 | if sys.version_info >= (3, 12): |
| 272 | return context.lib_path |
| 273 | |
| 274 | # Python 3.10+ |
| 275 | if "venv" in sysconfig.get_scheme_names(): |
| 276 | lib_path = sysconfig.get_path( |
| 277 | "purelib", scheme="venv", vars={"base": context.env_dir} |
| 278 | ) |
| 279 | assert lib_path is not None |
| 280 | return lib_path |
| 281 | |
| 282 | # For Python <= 3.9 we need to hardcode this. Fortunately the |
| 283 | # code below was the same in Python 3.6-3.10, so there is only |
| 284 | # one case. |
| 285 | if sys.platform == "win32": |
| 286 | return os.path.join(context.env_dir, "Lib", "site-packages") |
| 287 | return os.path.join( |
| 288 | context.env_dir, |
| 289 | "lib", |
| 290 | "python%d.%d" % sys.version_info[:2], |
| 291 | "site-packages", |
| 292 | ) |
| 293 | |
| 294 | def ensure_directories(self, env_dir: DirType) -> SimpleNamespace: |
| 295 | logger.debug("ensure_directories(env_dir=%s)", env_dir) |
| 296 | self._context = super().ensure_directories(env_dir) |
| 297 | return self._context |
| 298 | |
| 299 | def create(self, env_dir: DirType) -> None: |
| 300 | logger.debug("create(env_dir=%s)", env_dir) |
| 301 | super().create(env_dir) |
| 302 | assert self._context is not None |
| 303 | self.post_post_setup(self._context) |
| 304 | |
| 305 | def post_post_setup(self, context: SimpleNamespace) -> None: |
| 306 | """ |
| 307 | The final, final hook. Enter the venv and run commands inside of it. |
| 308 | """ |
| 309 | if self.use_parent_packages: |
| 310 | # We're inside of a venv and we want to include the parent |
| 311 | # venv's packages. |
| 312 | parent_libpath = self.get_parent_libpath() |
| 313 | assert parent_libpath is not None |
| 314 | logger.debug("parent_libpath: %s", parent_libpath) |
| 315 | |
| 316 | our_libpath = self.compute_venv_libpath(context) |
| 317 | logger.debug("our_libpath: %s", our_libpath) |
| 318 | |
| 319 | pth_file = os.path.join(our_libpath, "nested.pth") |
| 320 | with open(pth_file, "w", encoding="UTF-8") as file: |
| 321 | file.write(parent_libpath + os.linesep) |
| 322 | |
| 323 | if self.want_pip: |
| 324 | args = [ |
| 325 | context.env_exe, |
| 326 | __file__, |
| 327 | "post_init", |
| 328 | ] |
| 329 | subprocess.run(args, check=True) |
| 330 | |
| 331 | def get_value(self, field: str) -> str: |
| 332 | """ |
| 333 | Get a string value from the context namespace after a call to build. |
| 334 | |
| 335 | For valid field names, see: |
| 336 | https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories |
| 337 | """ |
| 338 | ret = getattr(self._context, field) |
| 339 | assert isinstance(ret, str) |
| 340 | return ret |
| 341 | |
| 342 | |
| 343 | def need_ensurepip() -> bool: |
| 344 | """ |
| 345 | Tests for the presence of setuptools and pip. |
| 346 | |
| 347 | :return: `True` if we do not detect both packages. |
| 348 | """ |
| 349 | # Don't try to actually import them, it's fraught with danger: |
| 350 | # https://github.com/pypa/setuptools/issues/2993 |
| 351 | if find_spec("setuptools") and find_spec("pip"): |
| 352 | return False |
| 353 | return True |
| 354 | |
| 355 | |
| 356 | def check_ensurepip() -> None: |
| 357 | """ |
| 358 | Check that we have ensurepip. |
| 359 | |
| 360 | Raise a fatal exception with a helpful hint if it isn't available. |
| 361 | """ |
| 362 | if not find_spec("ensurepip"): |
| 363 | msg = ( |
| 364 | "Python's ensurepip module is not found.\n" |
| 365 | "It's normally part of the Python standard library, " |
| 366 | "maybe your distribution packages it separately?\n" |
| 367 | "Either install ensurepip, or alleviate the need for it in the " |
| 368 | "first place by installing pip and setuptools for " |
| 369 | f"'{sys.executable}'.\n" |
| 370 | "(Hint: Debian puts ensurepip in its python3-venv package.)" |
| 371 | ) |
| 372 | raise Ouch(msg) |
| 373 | |
| 374 | # ensurepip uses pyexpat, which can also go missing on us: |
| 375 | if not find_spec("pyexpat"): |
| 376 | msg = ( |
| 377 | "Python's pyexpat module is not found.\n" |
| 378 | "It's normally part of the Python standard library, " |
| 379 | "maybe your distribution packages it separately?\n" |
| 380 | "Either install pyexpat, or alleviate the need for it in the " |
| 381 | "first place by installing pip and setuptools for " |
| 382 | f"'{sys.executable}'.\n\n" |
| 383 | "(Hint: NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)" |
| 384 | ) |
| 385 | raise Ouch(msg) |
| 386 | |
| 387 | |
| 388 | def make_venv( # pylint: disable=too-many-arguments |
| 389 | env_dir: Union[str, Path], |
| 390 | system_site_packages: bool = False, |
| 391 | clear: bool = True, |
| 392 | symlinks: Optional[bool] = None, |
| 393 | with_pip: bool = True, |
| 394 | ) -> None: |
| 395 | """ |
| 396 | Create a venv using `QemuEnvBuilder`. |
| 397 | |
| 398 | This is analogous to the `venv.create` module-level convenience |
| 399 | function that is part of the Python stdblib, except it uses |
| 400 | `QemuEnvBuilder` instead. |
| 401 | |
| 402 | :param env_dir: The directory to create/install to. |
| 403 | :param system_site_packages: |
| 404 | Allow inheriting packages from the system installation. |
| 405 | :param clear: When True, fully remove any prior venv and files. |
| 406 | :param symlinks: |
| 407 | Whether to use symlinks to the target interpreter or not. If |
| 408 | left unspecified, it will use symlinks except on Windows to |
| 409 | match behavior with the "venv" CLI tool. |
| 410 | :param with_pip: |
| 411 | Whether to install "pip" binaries or not. |
| 412 | """ |
| 413 | logger.debug( |
| 414 | "%s: make_venv(env_dir=%s, system_site_packages=%s, " |
| 415 | "clear=%s, symlinks=%s, with_pip=%s)", |
| 416 | __file__, |
| 417 | str(env_dir), |
| 418 | system_site_packages, |
| 419 | clear, |
| 420 | symlinks, |
| 421 | with_pip, |
| 422 | ) |
| 423 | |
| 424 | if symlinks is None: |
| 425 | # Default behavior of standard venv CLI |
| 426 | symlinks = os.name != "nt" |
| 427 | |
| 428 | builder = QemuEnvBuilder( |
| 429 | system_site_packages=system_site_packages, |
| 430 | clear=clear, |
| 431 | symlinks=symlinks, |
| 432 | with_pip=with_pip, |
| 433 | ) |
| 434 | |
| 435 | style = "non-isolated" if builder.system_site_packages else "isolated" |
| 436 | nested = "" |
| 437 | if builder.use_parent_packages: |
| 438 | nested = f"(with packages from '{builder.get_parent_libpath()}') " |
| 439 | print( |
| 440 | f"mkvenv: Creating {style} virtual environment" |
| 441 | f" {nested}at '{str(env_dir)}'", |
| 442 | file=sys.stderr, |
| 443 | ) |
| 444 | |
| 445 | try: |
| 446 | logger.debug("Invoking builder.create()") |
| 447 | try: |
| 448 | builder.create(str(env_dir)) |
| 449 | except SystemExit as exc: |
| 450 | # pylint 3.3 bug: |
| 451 | # pylint: disable=raising-non-exception, raise-missing-from |
| 452 | |
| 453 | # Some versions of the venv module raise SystemExit; *nasty*! |
| 454 | # We want the exception that prompted it. It might be a subprocess |
| 455 | # error that has output we *really* want to see. |
| 456 | logger.debug("Intercepted SystemExit from EnvBuilder.create()") |
| 457 | raise exc.__cause__ or exc.__context__ or exc |
| 458 | logger.debug("builder.create() finished") |
| 459 | except subprocess.CalledProcessError as exc: |
| 460 | logger.error("mkvenv subprocess failed:") |
| 461 | logger.error("cmd: %s", exc.cmd) |
| 462 | logger.error("returncode: %d", exc.returncode) |
| 463 | |
| 464 | def _stringify(data: Union[str, bytes]) -> str: |
| 465 | if isinstance(data, bytes): |
| 466 | return data.decode() |
| 467 | return data |
| 468 | |
| 469 | lines = [] |
| 470 | if exc.stdout: |
| 471 | lines.append("========== stdout ==========") |
| 472 | lines.append(_stringify(exc.stdout)) |
| 473 | lines.append("============================") |
| 474 | if exc.stderr: |
| 475 | lines.append("========== stderr ==========") |
| 476 | lines.append(_stringify(exc.stderr)) |
| 477 | lines.append("============================") |
| 478 | if lines: |
| 479 | logger.error(os.linesep.join(lines)) |
| 480 | |
| 481 | raise Ouch("VENV creation subprocess failed.") from exc |
| 482 | |
| 483 | # print the python executable to stdout for configure. |
| 484 | print(builder.get_value("env_exe")) |
| 485 | |
| 486 | |
| 487 | def _get_entry_points(packages: Sequence[str]) -> Iterator[str]: |
| 488 | |
| 489 | def _generator() -> Iterator[str]: |
| 490 | for package in packages: |
| 491 | try: |
| 492 | entry_points: Iterator[EntryPoint] = \ |
| 493 | iter(distribution(package).entry_points) |
| 494 | except PackageNotFoundError: |
| 495 | continue |
| 496 | |
| 497 | # The EntryPoints type is only available in 3.10+, |
| 498 | # treat this as a vanilla list and filter it ourselves. |
| 499 | entry_points = filter( |
| 500 | lambda ep: ep.group == "console_scripts", entry_points |
| 501 | ) |
| 502 | |
| 503 | for entry_point in entry_points: |
| 504 | yield f"{entry_point.name} = {entry_point.value}" |
| 505 | |
| 506 | return _generator() |
| 507 | |
| 508 | |
| 509 | def generate_console_scripts( |
| 510 | packages: Sequence[str], |
| 511 | python_path: Optional[str] = None, |
| 512 | bin_path: Optional[str] = None, |
| 513 | ) -> None: |
| 514 | """ |
| 515 | Generate script shims for console_script entry points in @packages. |
| 516 | """ |
| 517 | if python_path is None: |
| 518 | python_path = sys.executable |
| 519 | if bin_path is None: |
| 520 | bin_path = sysconfig.get_path("scripts") |
| 521 | assert bin_path is not None |
| 522 | |
| 523 | logger.debug( |
| 524 | "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)", |
| 525 | packages, |
| 526 | python_path, |
| 527 | bin_path, |
| 528 | ) |
| 529 | |
| 530 | if not packages: |
| 531 | return |
| 532 | |
| 533 | maker = distlib.scripts.ScriptMaker(None, bin_path) |
| 534 | maker.variants = {""} |
| 535 | maker.clobber = False |
| 536 | |
| 537 | for entry_point in _get_entry_points(packages): |
| 538 | for filename in maker.make(entry_point): |
| 539 | logger.debug("wrote console_script '%s'", filename) |
| 540 | |
| 541 | |
| 542 | def pkgname_from_depspec(dep_spec: str) -> str: |
| 543 | """ |
| 544 | Parse package name out of a PEP-508 depspec. |
| 545 | |
| 546 | See https://peps.python.org/pep-0508/#names |
| 547 | """ |
| 548 | match = re.match( |
| 549 | r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE |
| 550 | ) |
| 551 | if not match: |
| 552 | raise ValueError( |
| 553 | f"dep_spec '{dep_spec}'" |
| 554 | " does not appear to contain a valid package name" |
| 555 | ) |
| 556 | return match.group(0) |
| 557 | |
| 558 | |
| 559 | def _path_is_prefix(prefix: Optional[str], path: str) -> bool: |
| 560 | try: |
| 561 | return ( |
| 562 | prefix is not None and os.path.commonpath([prefix, path]) == prefix |
| 563 | ) |
| 564 | except ValueError: |
| 565 | return False |
| 566 | |
| 567 | |
| 568 | def _is_system_package(dist: Distribution) -> bool: |
| 569 | path = str(dist.locate_file(".")) |
| 570 | return not ( |
| 571 | _path_is_prefix(sysconfig.get_path("purelib"), path) |
| 572 | or _path_is_prefix(sysconfig.get_path("platlib"), path) |
| 573 | ) |
| 574 | |
| 575 | |
| 576 | def diagnose( |
| 577 | dep_spec: str, |
| 578 | online: bool, |
| 579 | wheels_dir: Optional[Union[str, Path]], |
| 580 | prog: Optional[str], |
| 581 | ) -> Tuple[str, bool]: |
| 582 | """ |
| 583 | Offer a summary to the user as to why a package failed to be installed. |
| 584 | |
| 585 | :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5' |
| 586 | :param online: Did we allow PyPI access? |
| 587 | :param prog: |
| 588 | Optionally, a shell program name that can be used as a |
| 589 | bellwether to detect if this program is installed elsewhere on |
| 590 | the system. This is used to offer advice when a program is |
| 591 | detected for a different python version. |
| 592 | :param wheels_dir: |
| 593 | Optionally, a directory that was searched for vendored packages. |
| 594 | """ |
| 595 | # pylint: disable=too-many-branches |
| 596 | |
| 597 | # Some errors are not particularly serious |
| 598 | bad = False |
| 599 | |
| 600 | pkg_name = pkgname_from_depspec(dep_spec) |
| 601 | pkg_version: Optional[str] = None |
| 602 | try: |
| 603 | pkg_version = version(pkg_name) |
| 604 | except PackageNotFoundError: |
| 605 | pass |
| 606 | |
| 607 | lines = [] |
| 608 | |
| 609 | if pkg_version: |
| 610 | lines.append( |
| 611 | f"Python package '{pkg_name}' version '{pkg_version}' was found," |
| 612 | " but isn't suitable." |
| 613 | ) |
| 614 | else: |
| 615 | lines.append( |
| 616 | f"Python package '{pkg_name}' was not found nor installed." |
| 617 | ) |
| 618 | |
| 619 | if wheels_dir: |
| 620 | lines.append( |
| 621 | "No suitable version found in, or failed to install from" |
| 622 | f" '{wheels_dir}'." |
| 623 | ) |
| 624 | bad = True |
| 625 | |
| 626 | if online: |
| 627 | lines.append("A suitable version could not be obtained from PyPI.") |
| 628 | bad = True |
| 629 | else: |
| 630 | lines.append( |
| 631 | "mkvenv was configured to operate offline and did not check PyPI." |
| 632 | ) |
| 633 | |
| 634 | if prog and not pkg_version: |
| 635 | which = shutil.which(prog) |
| 636 | if which: |
| 637 | if sys.base_prefix in site.PREFIXES: |
| 638 | pypath = Path(sys.executable).resolve() |
| 639 | lines.append( |
| 640 | f"'{prog}' was detected on your system at '{which}', " |
| 641 | f"but the Python package '{pkg_name}' was not found by " |
| 642 | f"this Python interpreter ('{pypath}'). " |
| 643 | f"Typically this means that '{prog}' has been installed " |
| 644 | "against a different Python interpreter on your system." |
| 645 | ) |
| 646 | else: |
| 647 | lines.append( |
| 648 | f"'{prog}' was detected on your system at '{which}', " |
| 649 | "but the build is using an isolated virtual environment." |
| 650 | ) |
| 651 | bad = True |
| 652 | |
| 653 | lines = [f" • {line}" for line in lines] |
| 654 | if bad: |
| 655 | lines.insert(0, f"Could not provide build dependency '{dep_spec}':") |
| 656 | else: |
| 657 | lines.insert(0, f"'{dep_spec}' not found:") |
| 658 | return os.linesep.join(lines), bad |
| 659 | |
| 660 | |
| 661 | def pip_install( |
| 662 | args: Sequence[str], |
| 663 | online: bool = False, |
| 664 | wheels_dir: Optional[Union[str, Path]] = None, |
| 665 | env: Optional[Dict[str, str]] = None, |
| 666 | ) -> None: |
| 667 | """ |
| 668 | Use pip to install a package or package(s) as specified in @args. |
| 669 | """ |
| 670 | loud = bool( |
| 671 | os.environ.get("DEBUG") |
| 672 | or os.environ.get("GITLAB_CI") |
| 673 | or os.environ.get("V") |
| 674 | ) |
| 675 | |
| 676 | full_args = [ |
| 677 | sys.executable, |
| 678 | "-m", |
| 679 | "pip", |
| 680 | "install", |
| 681 | "--disable-pip-version-check", |
| 682 | "-v" if loud else "-q", |
| 683 | ] |
| 684 | if not online: |
| 685 | full_args += ["--no-index"] |
| 686 | if wheels_dir: |
| 687 | full_args += ["--find-links", str(wheels_dir)] |
| 688 | full_args += list(args) |
| 689 | subprocess.run( |
| 690 | full_args, |
| 691 | env=env, |
| 692 | check=True, |
| 693 | ) |
| 694 | |
| 695 | |
| 696 | def _make_version_constraint(info: Dict[str, str], install: bool) -> str: |
| 697 | """ |
| 698 | Construct the version constraint part of a PEP 508 dependency |
| 699 | specification (for example '>=0.61.5') from the accepted and |
| 700 | installed keys of the provided dictionary. |
| 701 | |
| 702 | :param info: A dictionary corresponding to a TOML key-value list. |
| 703 | :param install: True generates install constraints, False generates |
| 704 | presence constraints |
| 705 | """ |
| 706 | if install and "installed" in info: |
| 707 | return "==" + info["installed"] |
| 708 | |
| 709 | dep_spec = info.get("accepted", "") |
| 710 | dep_spec = dep_spec.strip() |
| 711 | # Double check that they didn't just use a version number |
| 712 | if dep_spec and dep_spec[0] not in "!~><=(": |
| 713 | raise Ouch( |
| 714 | "invalid dependency specifier " + dep_spec + " in dependency file" |
| 715 | ) |
| 716 | |
| 717 | return dep_spec |
| 718 | |
| 719 | |
| 720 | def _do_ensure( |
| 721 | group: Dict[str, Dict[str, str]], |
| 722 | online: bool = False, |
| 723 | wheels_dir: Optional[Union[str, Path]] = None, |
| 724 | ) -> Optional[Tuple[str, bool]]: |
| 725 | """ |
| 726 | Use pip to ensure we have the packages specified in @group. |
| 727 | |
| 728 | If the packages are already installed, do nothing. If online and |
| 729 | wheels_dir are both provided, prefer packages found in wheels_dir |
| 730 | first before connecting to PyPI. |
| 731 | |
| 732 | :param group: A dictionary of dictionaries, corresponding to a |
| 733 | section in a pythondeps.toml file. |
| 734 | :param online: If True, fall back to PyPI. |
| 735 | :param wheels_dir: If specified, search this path for packages. |
| 736 | """ |
| 737 | absent = [] |
| 738 | local_packages = [] |
| 739 | present = [] |
| 740 | canary = None |
| 741 | for name, info in group.items(): |
| 742 | if "path" in info: |
| 743 | pkgpath = Path(__file__).parents[2].joinpath(info["path"]) |
| 744 | local_packages.append(str(pkgpath)) |
| 745 | continue |
| 746 | constraint = _make_version_constraint(info, False) |
| 747 | matcher = Matcher(name + constraint) |
| 748 | print(f"mkvenv: checking for {matcher}", file=sys.stderr) |
| 749 | |
| 750 | dist: Optional[Distribution] = None |
| 751 | try: |
| 752 | dist = distribution(matcher.name) |
| 753 | except PackageNotFoundError: |
| 754 | pass |
| 755 | |
| 756 | if ( |
| 757 | dist is None |
| 758 | # Always pass installed package to pip, so that they can be |
| 759 | # updated if the requested version changes |
| 760 | or not _is_system_package(dist) |
| 761 | or not matcher.match(dist.version) |
| 762 | ): |
| 763 | absent.append(name + _make_version_constraint(info, True)) |
| 764 | if len(absent) == 1: |
| 765 | canary = info.get("canary", None) |
| 766 | else: |
| 767 | logger.info("found %s %s", name, dist.version) |
| 768 | present.append(name) |
| 769 | |
| 770 | if present: |
| 771 | generate_console_scripts(present) |
| 772 | |
| 773 | if absent: |
| 774 | if online or wheels_dir: |
| 775 | # Some packages are missing or aren't a suitable version, |
| 776 | # install a suitable (possibly vendored) package. |
| 777 | print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr) |
| 778 | try: |
| 779 | pip_install(args=absent, online=online, wheels_dir=wheels_dir) |
| 780 | absent = [] |
| 781 | except subprocess.CalledProcessError: |
| 782 | pass |
| 783 | |
| 784 | if absent: |
| 785 | return diagnose( |
| 786 | absent[0], |
| 787 | online, |
| 788 | wheels_dir, |
| 789 | canary, |
| 790 | ) |
| 791 | |
| 792 | # Handle local packages separately and last so we can use different |
| 793 | # installation arguments (-e), and so that any dependencies that may |
| 794 | # be covered above will be handled according to the depfile |
| 795 | # specifications. |
| 796 | if local_packages: |
| 797 | print(f"mkvenv: installing {', '.join(local_packages)}", |
| 798 | file=sys.stderr) |
| 799 | env = dict(os.environ) |
| 800 | env['PIP_CONFIG_SETTINGS'] = "editable_mode=compat" |
| 801 | pip_install( |
| 802 | args=["--no-build-isolation", |
| 803 | "-e"] + local_packages, |
| 804 | online=online, |
| 805 | wheels_dir=wheels_dir, |
| 806 | env=env, |
| 807 | ) |
| 808 | |
| 809 | return None |
| 810 | |
| 811 | |
| 812 | def _parse_groups(file: str) -> Dict[str, Dict[str, Any]]: |
| 813 | if not HAVE_TOMLLIB: |
| 814 | if sys.version_info < (3, 11): |
| 815 | raise Ouch("found no usable tomli, please install it") |
| 816 | |
| 817 | raise Ouch( |
| 818 | "Python >=3.11 does not have tomllib... what have you done!?" |
| 819 | ) |
| 820 | |
| 821 | # Use loads() to support both tomli v1.2.x (Ubuntu 22.04, |
| 822 | # Debian bullseye-backports) and v2.0.x |
| 823 | with open(file, "r", encoding="ascii") as depfile: |
| 824 | contents = depfile.read() |
| 825 | return tomllib.loads(contents) # type: ignore |
| 826 | |
| 827 | |
| 828 | def ensure_group( |
| 829 | file: str, |
| 830 | groups: Sequence[str], |
| 831 | online: bool = False, |
| 832 | wheels_dir: Optional[Union[str, Path]] = None, |
| 833 | ) -> None: |
| 834 | """ |
| 835 | Use pip to ensure we have the package specified by @dep_specs. |
| 836 | |
| 837 | If the package is already installed, do nothing. If online and |
| 838 | wheels_dir are both provided, prefer packages found in wheels_dir |
| 839 | first before connecting to PyPI. |
| 840 | |
| 841 | :param dep_specs: |
| 842 | PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. |
| 843 | :param online: If True, fall back to PyPI. |
| 844 | :param wheels_dir: If specified, search this path for packages. |
| 845 | """ |
| 846 | |
| 847 | if not HAVE_DISTLIB: |
| 848 | raise Ouch("found no usable distlib, please install it") |
| 849 | |
| 850 | parsed_deps = _parse_groups(file) |
| 851 | |
| 852 | to_install: Dict[str, Dict[str, str]] = {} |
| 853 | for group in groups: |
| 854 | try: |
| 855 | to_install.update(parsed_deps[group]) |
| 856 | except KeyError as exc: |
| 857 | raise Ouch(f"group {group} not defined") from exc |
| 858 | |
| 859 | result = _do_ensure(to_install, online, wheels_dir) |
| 860 | if result: |
| 861 | # Well, that's not good. |
| 862 | if result[1]: |
| 863 | raise Ouch(result[0]) |
| 864 | raise SystemExit(f"\n{result[0]}\n\n") |
| 865 | |
| 866 | if inside_a_venv(): |
| 867 | for group in groups: |
| 868 | path = Path(sys.prefix).joinpath(f"{group}.group") |
| 869 | with open(path, "w", encoding="UTF8"): |
| 870 | pass |
| 871 | |
| 872 | |
| 873 | def post_venv_setup() -> None: |
| 874 | """ |
| 875 | This is intended to be run *inside the venv* after it is created. |
| 876 | """ |
| 877 | logger.debug("post_venv_setup()") |
| 878 | # Generate a 'pip' script so the venv is usable in a normal |
| 879 | # way from the CLI. This only happens when we inherited pip from a |
| 880 | # parent/system-site and haven't run ensurepip in some way. |
| 881 | generate_console_scripts(["pip"]) |
| 882 | |
| 883 | |
| 884 | def _add_create_subcommand(subparsers: Any) -> None: |
| 885 | subparser = subparsers.add_parser("create", help="create a venv") |
| 886 | subparser.add_argument( |
| 887 | "target", |
| 888 | type=str, |
| 889 | action="store", |
| 890 | help="Target directory to install virtual environment into.", |
| 891 | ) |
| 892 | |
| 893 | |
| 894 | def _add_post_init_subcommand(subparsers: Any) -> None: |
| 895 | subparsers.add_parser("post_init", help="post-venv initialization") |
| 896 | |
| 897 | |
| 898 | def _add_ensuregroup_subcommand(subparsers: Any) -> None: |
| 899 | subparser = subparsers.add_parser( |
| 900 | "ensuregroup", |
| 901 | help="Ensure that the specified package group is installed.", |
| 902 | ) |
| 903 | subparser.add_argument( |
| 904 | "--online", |
| 905 | action="store_true", |
| 906 | help="Install packages from PyPI, if necessary.", |
| 907 | ) |
| 908 | subparser.add_argument( |
| 909 | "--dir", |
| 910 | type=str, |
| 911 | action="store", |
| 912 | help="Path to vendored packages where we may install from.", |
| 913 | ) |
| 914 | subparser.add_argument( |
| 915 | "file", |
| 916 | type=str, |
| 917 | action="store", |
| 918 | help=("Path to a TOML file describing package groups"), |
| 919 | ) |
| 920 | subparser.add_argument( |
| 921 | "group", |
| 922 | type=str, |
| 923 | action="store", |
| 924 | help="One or more package group names", |
| 925 | nargs="+", |
| 926 | ) |
| 927 | |
| 928 | |
| 929 | def main() -> int: |
| 930 | """CLI interface to make_qemu_venv. See module docstring.""" |
| 931 | if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): |
| 932 | # You're welcome. |
| 933 | logging.basicConfig(level=logging.DEBUG) |
| 934 | else: |
| 935 | if os.environ.get("V"): |
| 936 | logging.basicConfig(level=logging.INFO) |
| 937 | |
| 938 | parser = argparse.ArgumentParser( |
| 939 | prog="mkvenv", |
| 940 | description="QEMU pyvenv bootstrapping utility", |
| 941 | ) |
| 942 | subparsers = parser.add_subparsers( |
| 943 | title="Commands", |
| 944 | dest="command", |
| 945 | required=True, |
| 946 | metavar="command", |
| 947 | help="Description", |
| 948 | ) |
| 949 | |
| 950 | _add_create_subcommand(subparsers) |
| 951 | _add_post_init_subcommand(subparsers) |
| 952 | _add_ensuregroup_subcommand(subparsers) |
| 953 | |
| 954 | args = parser.parse_args() |
| 955 | try: |
| 956 | if args.command == "create": |
| 957 | make_venv( |
| 958 | args.target, |
| 959 | system_site_packages=True, |
| 960 | clear=True, |
| 961 | ) |
| 962 | if args.command == "post_init": |
| 963 | post_venv_setup() |
| 964 | if args.command == "ensuregroup": |
| 965 | ensure_group( |
| 966 | file=args.file, |
| 967 | groups=args.group, |
| 968 | online=args.online, |
| 969 | wheels_dir=args.dir, |
| 970 | ) |
| 971 | logger.debug("mkvenv.py %s: exiting", args.command) |
| 972 | except Ouch as exc: |
| 973 | print("\n*** Ouch! ***\n", file=sys.stderr) |
| 974 | print(str(exc), "\n\n", file=sys.stderr) |
| 975 | return 1 |
| 976 | except SystemExit: |
| 977 | raise |
| 978 | except: # pylint: disable=bare-except |
| 979 | logger.exception("mkvenv did not complete successfully:") |
| 980 | return 2 |
| 981 | return 0 |
| 982 | |
| 983 | |
| 984 | if __name__ == "__main__": |
| 985 | sys.exit(main()) |