A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bugfix] Fix using `-O[0,3]` with LLM entrypoint (#10677)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Nov 26, 2024 at 13:44 UTC
9a99273b482a3e90431069f37858d60827983e2f
2 files changed
+12
-3
vllm/engine/arg_utils.py
+4
-1
@@ -206,7 +206,10 @@ class EngineArgs:
206
# support `EngineArgs(compilation_config={...})`
207
# without having to manually construct a
208
# CompilationConfig object
209
- if isinstance(self.compilation_config, (int, dict)):
209
+ if isinstance(self.compilation_config, (int)):
210
+ self.compilation_config = CompilationConfig.from_cli(
211
+ str(self.compilation_config))
212
+ elif isinstance(self.compilation_config, (dict)):
213
self.compilation_config = CompilationConfig.from_cli(
214
json.dumps(self.compilation_config))
215
vllm/entrypoints/llm.py
+8
-2
@@ -185,8 +185,14 @@ class LLM:
185
kwargs["disable_log_stats"] = True
186
187
if compilation_config is not None:
188
- compilation_config_instance = CompilationConfig.from_cli(
189
- json.dumps(compilation_config))
188
+ if isinstance(compilation_config, (int)):
189
+ compilation_config_instance = CompilationConfig.from_cli(
190
+ str(compilation_config))
191
+ elif isinstance(compilation_config, (dict)):
192
+ compilation_config_instance = CompilationConfig.from_cli(
193
+ json.dumps(compilation_config))
194
+ else:
195
+ compilation_config_instance = compilation_config
196
else:
197
compilation_config_instance = None
198