A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[torch.compile] upgrade tests (#9858)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Oct 30, 2024 at 16:34 UTC
64384bbcdfe6bdf4b50ff82bda90e728160325f5
1 file changed
+14
-12
tests/compile/test_basic_correctness.py
+14
-12
@@ -30,18 +30,20 @@ def test_compile_correctness(model, model_args, pp_size, tp_size, attn_backend,
30
pytest.skip("Not correct CUDA devices for the test.")
31
import os
32
os.environ["VLLM_ATTENTION_BACKEND"] = attn_backend
33
- if not fullgraph:
34
- os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "0"
35
- all_args = [["--enforce-eager"] + model_args + ["--max_model_len", "1024"]
36
- + ["-pp", str(pp_size)] + ["-tp", str(tp_size)]] * 3
33
+ all_args = [["--enforce-eager"] + model_args + ["-pp", str(pp_size)] +
34
+ ["-tp", str(tp_size)]] * 3
35
# don't test VLLM_TORCH_COMPILE_LEVEL == 3 case
36
# inductor will change the output, so we cannot compare them.
39
- all_envs: List[Optional[Dict[str, str]]] = [{
40
- "VLLM_TORCH_COMPILE_LEVEL":
41
- str(level)
42
- } for level in [
43
- CompilationLevel.NO_COMPILATION,
44
- CompilationLevel.DYNAMO_AS_IS,
45
- CompilationLevel.DYNAMO_ONCE,
46
- ]]
37
+ all_envs: List[Optional[Dict[str, str]]] = []
38
+ for level in [
39
+ CompilationLevel.NO_COMPILATION,
40
+ CompilationLevel.DYNAMO_AS_IS,
41
+ CompilationLevel.DYNAMO_ONCE,
42
+ ]:
43
+ all_envs.append({"VLLM_TORCH_COMPILE_LEVEL": str(level)})
44
+ if level != CompilationLevel.DYNAMO_ONCE and not fullgraph:
45
+ # "DYNAMO_ONCE" will always use fullgraph
46
+ all_envs[-1][
47
+ "VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "0" # type: ignore
48
+
49
compare_all_settings(model, all_args, all_envs, method=method)