@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9a99273b48
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Bugfix] Fix using `-O[0,3]` with LLM entrypoint (#10677)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Nov 26, 2024 at 13:44 UTC 9a99273b482a3e90431069f37858d60827983e2f
2 files changed +12 -3
vllm/engine/arg_utils.py
+4 -1
@@ -206,7 +206,10 @@ class EngineArgs:
206 # support `EngineArgs(compilation_config={...})`
207 # without having to manually construct a
208 # CompilationConfig object
209 - if isinstance(self.compilation_config, (int, dict)):
209 + if isinstance(self.compilation_config, (int)):
210 + self.compilation_config = CompilationConfig.from_cli(
211 + str(self.compilation_config))
212 + elif isinstance(self.compilation_config, (dict)):
213 self.compilation_config = CompilationConfig.from_cli(
214 json.dumps(self.compilation_config))
215
vllm/entrypoints/llm.py
+8 -2
@@ -185,8 +185,14 @@ class LLM:
185 kwargs["disable_log_stats"] = True
186
187 if compilation_config is not None:
188 - compilation_config_instance = CompilationConfig.from_cli(
189 - json.dumps(compilation_config))
188 + if isinstance(compilation_config, (int)):
189 + compilation_config_instance = CompilationConfig.from_cli(
190 + str(compilation_config))
191 + elif isinstance(compilation_config, (dict)):
192 + compilation_config_instance = CompilationConfig.from_cli(
193 + json.dumps(compilation_config))
194 + else:
195 + compilation_config_instance = compilation_config
196 else:
197 compilation_config_instance = None
198