@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / db100c5cde
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[bugfix] fix full graph tests (#10581)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 22, 2024 at 10:02 UTC db100c5cdebc7140b57cbb40b20b5a28d7bff386
1 file changed +2 -2
tests/compile/utils.py
+2 -2
@@ -4,7 +4,7 @@ import torch
4
5 from tests.quantization.utils import is_quant_method_supported
6 from vllm import LLM, SamplingParams
7 -from vllm.config import CompilationConfig, CompilationLevel
7 +from vllm.config import CompilationLevel
8 from vllm.platforms import current_platform
9
10 TEST_MODELS = [
@@ -85,7 +85,7 @@ def check_full_graph_support(model,
85 enforce_eager=True,
86 tensor_parallel_size=tp_size,
87 disable_custom_all_reduce=True,
88 - compilation_config=CompilationConfig(level=optimization_level),
88 + compilation_config=optimization_level,
89 **model_kwargs)
90
91 outputs = llm.generate(prompts, sampling_params)