A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 15:45 UTC
[Bugfix] Fix render server crash for quantized models on CPU-only hosts (#37215)
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
Sage committed
Mar 16, 2026 at 20:59 UTC
0fefd00e6ccf6670686eb2cc0a5eda57f56e625a
1 file changed
+5
vllm/entrypoints/cli/launch.py
+5
@@ -116,6 +116,11 @@ async def run_launch_fastapi(args: argparse.Namespace) -> None:
116
# 2. Build and serve the API server
117
engine_args = AsyncEngineArgs.from_cli_args(args)
118
model_config = engine_args.create_model_config()
119
+
120
+ # Render servers preprocess data only — no inference, no quantized kernels.
121
+ # Clear quantization so VllmConfig skips quant dtype/capability validation.
122
+ model_config.quantization = None
123
+
124
vllm_config = VllmConfig(model_config=model_config)
125
shutdown_task = await build_and_serve_renderer(
126
vllm_config, listen_address, sock, args