A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[bugfix] fix full graph tests (#10581)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 22, 2024 at 10:02 UTC
db100c5cdebc7140b57cbb40b20b5a28d7bff386
1 file changed
+2
-2
tests/compile/utils.py
+2
-2
@@ -4,7 +4,7 @@ import torch
4
5
from tests.quantization.utils import is_quant_method_supported
6
from vllm import LLM, SamplingParams
7
-from vllm.config import CompilationConfig, CompilationLevel
7
+from vllm.config import CompilationLevel
8
from vllm.platforms import current_platform
9
10
TEST_MODELS = [
@@ -85,7 +85,7 @@ def check_full_graph_support(model,
85
enforce_eager=True,
86
tensor_parallel_size=tp_size,
87
disable_custom_all_reduce=True,
88
- compilation_config=CompilationConfig(level=optimization_level),
88
+ compilation_config=optimization_level,
89
**model_kwargs)
90
91
outputs = llm.generate(prompts, sampling_params)