A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[misc] add torch.compile compatibility check (#10618)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 24, 2024 at 23:40 UTC
25d806e95391a8556deb69bdb214714425f776c9
3 files changed
+22
-1
tests/v1/engine/test_engine_core_client.py
+1
-1
@@ -81,7 +81,7 @@ def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):
81
with monkeypatch.context() as m:
82
m.setenv("VLLM_USE_V1", "1")
83
84
- engine_args = EngineArgs(model=MODEL_NAME)
84
+ engine_args = EngineArgs(model=MODEL_NAME, compilation_config=3)
85
vllm_config = engine_args.create_engine_config()
86
executor_class = AsyncLLM._get_executor_cls(vllm_config)
87
client = EngineCoreClient.make_client(
vllm/config.py
+14
@@ -2394,6 +2394,20 @@ class VllmConfig:
2394
self.compilation_config.pass_config.enable_reshape = False
2395
self.compilation_config.level = CompilationLevel.PIECEWISE
2396
2397
+ if self.cache_config is not None and \
2398
+ self.cache_config.cpu_offload_gb > 0 and \
2399
+ self.compilation_config.level != CompilationLevel.NO_COMPILATION:
2400
+ logger.warning(
2401
+ "CPU offload is not supported with `torch.compile` yet."
2402
+ " Disabling `torch.compile`.")
2403
+ self.compilation_config.level = CompilationLevel.NO_COMPILATION
2404
+
2405
+ if self.lora_config is not None and self.compilation_config.level !=\
2406
+ CompilationLevel.NO_COMPILATION:
2407
+ logger.warning("LoRA is not supported with `torch.compile` yet. "
2408
+ "Disabling `torch.compile`.")
2409
+ self.compilation_config.level = CompilationLevel.NO_COMPILATION
2410
+
2411
current_platform.check_and_update_config(self)
2412
2413
def __str__(self):
vllm/engine/arg_utils.py
+7
@@ -197,6 +197,13 @@ class EngineArgs:
197
if not self.tokenizer:
198
self.tokenizer = self.model
199
200
+ # support `EngineArgs(compilation_config={...})`
201
+ # without having to manually construct a
202
+ # CompilationConfig object
203
+ if isinstance(self.compilation_config, (int, dict)):
204
+ self.compilation_config = CompilationConfig.from_cli(
205
+ json.dumps(self.compilation_config))
206
+
207
# Setup plugins
208
from vllm.plugins import load_general_plugins
209
load_general_plugins()