@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 25d806e953
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[misc] add torch.compile compatibility check (#10618)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 24, 2024 at 23:40 UTC 25d806e95391a8556deb69bdb214714425f776c9
3 files changed +22 -1
tests/v1/engine/test_engine_core_client.py
+1 -1
@@ -81,7 +81,7 @@ def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):
81 with monkeypatch.context() as m:
82 m.setenv("VLLM_USE_V1", "1")
83
84 - engine_args = EngineArgs(model=MODEL_NAME)
84 + engine_args = EngineArgs(model=MODEL_NAME, compilation_config=3)
85 vllm_config = engine_args.create_engine_config()
86 executor_class = AsyncLLM._get_executor_cls(vllm_config)
87 client = EngineCoreClient.make_client(
vllm/config.py
+14
@@ -2394,6 +2394,20 @@ class VllmConfig:
2394 self.compilation_config.pass_config.enable_reshape = False
2395 self.compilation_config.level = CompilationLevel.PIECEWISE
2396
2397 + if self.cache_config is not None and \
2398 + self.cache_config.cpu_offload_gb > 0 and \
2399 + self.compilation_config.level != CompilationLevel.NO_COMPILATION:
2400 + logger.warning(
2401 + "CPU offload is not supported with `torch.compile` yet."
2402 + " Disabling `torch.compile`.")
2403 + self.compilation_config.level = CompilationLevel.NO_COMPILATION
2404 +
2405 + if self.lora_config is not None and self.compilation_config.level !=\
2406 + CompilationLevel.NO_COMPILATION:
2407 + logger.warning("LoRA is not supported with `torch.compile` yet. "
2408 + "Disabling `torch.compile`.")
2409 + self.compilation_config.level = CompilationLevel.NO_COMPILATION
2410 +
2411 current_platform.check_and_update_config(self)
2412
2413 def __str__(self):
vllm/engine/arg_utils.py
+7
@@ -197,6 +197,13 @@ class EngineArgs:
197 if not self.tokenizer:
198 self.tokenizer = self.model
199
200 + # support `EngineArgs(compilation_config={...})`
201 + # without having to manually construct a
202 + # CompilationConfig object
203 + if isinstance(self.compilation_config, (int, dict)):
204 + self.compilation_config = CompilationConfig.from_cli(
205 + json.dumps(self.compilation_config))
206 +
207 # Setup plugins
208 from vllm.plugins import load_general_plugins
209 load_general_plugins()