@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5f6a02812a

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 4, 03:45 UTC

[CI][Bugfix] Fix failure CI step "PyTorch Fullgraph Smoke Test" (#41953)

Signed-off-by: haosdent <haosdent@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io>

haosdent committed May 8, 2026 at 10:41 UTC 5f6a02812ae127679d407ac44852baaeac7822a6

2 files changed +53 -15

tests/compile/fullgraph/test_simple.py

+8 -1

@@ -161,7 +161,14 @@ def _run_simple_model(

161 @pytest.mark.parametrize("intermediate_unbacked", [True, False])

162 @torch.inference_mode()

163 @create_new_process_for_each_test("spawn")

164 -def test_simple_piecewise_compile(backend, intermediate_unbacked):

164 +def test_simple_piecewise_compile(backend, intermediate_unbacked, monkeypatch):

165 + # `intermediate_unbacked` flips a control-flow branch inside

166 + # `SillyModel.forward`, but the AOT-compile cache key only hashes the

167 + # forward function's qualname + line number, so both parametrize variants

168 + # share the same cache slot. Disabling the cache forces each variant to

169 + # compile fresh; otherwise the second-running variant loads the first's

170 + # artifact and segfaults with an illegal memory access.

171 + monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")

172 _run_simple_model(

173 splitting_ops=["silly::attention"],

174 use_inductor_graph_partition=False,

tests/utils.py

+45 -14

@@ -1511,43 +1511,72 @@ def fork_new_process_for_each_test(func: Callable[_P, None]) -> Callable[_P, Non

1511 return wrapper

1512

1513

1514 +# Set on the spawn-child interpreter so the wrapper short-circuits when the

1515 +# child resolves `module.qualname` back to its own decorated form, instead of

1516 +# launching another subprocess.

1517 +_SPAWN_CHILD_ENV = "VLLM_TEST_SPAWN_CHILD"

1518 +

1519 +

1520 def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None]:

1521 """Decorator to spawn a new process for each test function.

1522

1517 - Uses subprocess with cloudpickle to serialize the test function and

1518 - propagates exceptions back to the parent, so test failures are never

1519 - silently swallowed (fixes https://github.com/vllm-project/vllm/issues/41415).

1523 + Uses subprocess to run each test in a fresh interpreter and propagates

1524 + exceptions back to the parent, so test failures are never silently

1525 + swallowed (fixes https://github.com/vllm-project/vllm/issues/41415).

1526 +

1527 + The child resolves the test function by importing its module and looking

1528 + it up by qualified name, rather than reconstructing it from a cloudpickle

1529 + blob. Pickling the function by value would also pickle its ``__globals__``

1530 + by value — turning module-level singletons (e.g.

1531 + ``vllm.compilation.counter.compilation_counter``) into stale clones in

1532 + the child, so increments performed by the production code in the child

1533 + would never be observable to the test.

1534 """

1535

1536 @functools.wraps(f)

1537 def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:

1538 + if os.environ.get(_SPAWN_CHILD_ENV) == "1":

1539 + return f(*args, **kwargs)

1540 +

1541 with tempfile.NamedTemporaryFile(delete=False, suffix=".tb", mode="wb") as tmp:

1542 tb_file = tmp.name

1543

1544 try:

1528 - # Serialize the function + args with cloudpickle so closures work

1529 - payload = cloudpickle.dumps((f, args, kwargs, tb_file))

1545 + payload = cloudpickle.dumps(

1546 + {

1547 + "module": f.__module__,

1548 + "qualname": f.__qualname__,

1549 + "args": args,

1550 + "kwargs": kwargs,

1551 + "tb_file": tb_file,

1552 + }

1553 + )

1554

1555 child_script = (

1532 - "import sys, cloudpickle, traceback\n"

1556 + "import sys, importlib, cloudpickle, traceback\n"

1557 "try:\n"

1558 " from _pytest.outcomes import Skipped\n"

1559 "except ImportError:\n"

1560 " class Skipped(BaseException): pass\n"

1537 - "f, args, kwargs, tb_file = "

1538 - "cloudpickle.loads(sys.stdin.buffer.read())\n"

1561 + "data = cloudpickle.loads(sys.stdin.buffer.read())\n"

1562 + "mod = importlib.import_module(data['module'])\n"

1563 + "target = mod\n"

1564 + "for name in data['qualname'].split('.'):\n"

1565 + " target = getattr(target, name)\n"

1566 "try:\n"

1540 - " f(*args, **kwargs)\n"

1567 + " target(*data['args'], **data['kwargs'])\n"

1568 "except Skipped:\n"

1569 " sys.exit(0)\n"

1570 "except BaseException:\n"

1544 - " open(tb_file, 'w').write(traceback.format_exc())\n"

1571 + " with open(data['tb_file'], 'w') as fp:\n"

1572 + " fp.write(traceback.format_exc())\n"

1573 " sys.exit(1)\n"

1574 )

1575

1576 repo_root = str(VLLM_PATH.resolve())

1577 env = os.environ.copy()

1578 env["PYTHONPATH"] = repo_root + os.pathsep + env.get("PYTHONPATH", "")

1579 + env[_SPAWN_CHILD_ENV] = "1"

1580

1581 result = subprocess.run(

1582 [sys.executable, "-c", child_script],

@@ -1557,12 +1586,14 @@ def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None]

1586 )

1587

1588 if result.returncode != 0:

1560 - # Read traceback written by child, fall back to stderr

1561 - tb = ""

1562 - if os.path.exists(tb_file) and os.path.getsize(tb_file) > 0:

1589 + # Prefer the child's traceback file; fall back to stderr if

1590 + # the child crashed before its except handler ran.

1591 + try:

1592 with open(tb_file) as fp:

1593 tb = fp.read()

1565 - else:

1594 + except OSError:

1595 + tb = ""

1596 + if not tb:

1597 tb = result.stderr.decode()

1598 raise RuntimeError(

1599 f"Test subprocess '{f.__name__}' failed "