@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 64384bbcdf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[torch.compile] upgrade tests (#9858)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Oct 30, 2024 at 16:34 UTC 64384bbcdfe6bdf4b50ff82bda90e728160325f5
1 file changed +14 -12
tests/compile/test_basic_correctness.py
+14 -12
@@ -30,18 +30,20 @@ def test_compile_correctness(model, model_args, pp_size, tp_size, attn_backend,
30 pytest.skip("Not correct CUDA devices for the test.")
31 import os
32 os.environ["VLLM_ATTENTION_BACKEND"] = attn_backend
33 - if not fullgraph:
34 - os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "0"
35 - all_args = [["--enforce-eager"] + model_args + ["--max_model_len", "1024"]
36 - + ["-pp", str(pp_size)] + ["-tp", str(tp_size)]] * 3
33 + all_args = [["--enforce-eager"] + model_args + ["-pp", str(pp_size)] +
34 + ["-tp", str(tp_size)]] * 3
35 # don't test VLLM_TORCH_COMPILE_LEVEL == 3 case
36 # inductor will change the output, so we cannot compare them.
39 - all_envs: List[Optional[Dict[str, str]]] = [{
40 - "VLLM_TORCH_COMPILE_LEVEL":
41 - str(level)
42 - } for level in [
43 - CompilationLevel.NO_COMPILATION,
44 - CompilationLevel.DYNAMO_AS_IS,
45 - CompilationLevel.DYNAMO_ONCE,
46 - ]]
37 + all_envs: List[Optional[Dict[str, str]]] = []
38 + for level in [
39 + CompilationLevel.NO_COMPILATION,
40 + CompilationLevel.DYNAMO_AS_IS,
41 + CompilationLevel.DYNAMO_ONCE,
42 + ]:
43 + all_envs.append({"VLLM_TORCH_COMPILE_LEVEL": str(level)})
44 + if level != CompilationLevel.DYNAMO_ONCE and not fullgraph:
45 + # "DYNAMO_ONCE" will always use fullgraph
46 + all_envs[-1][
47 + "VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "0" # type: ignore
48 +
49 compare_all_settings(model, all_args, all_envs, method=method)