@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c1c0d00b88
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Don't use cupy when `enforce_eager=True` (#3037)

Roy committed Feb 27, 2024 at 09:33 UTC c1c0d00b88320f97e00a3175fac235a232893da5
1 file changed +4 -1
vllm/engine/llm_engine.py
+4 -1
@@ -284,7 +284,10 @@ class LLMEngine:
284 is_driver_worker=True,
285 )
286
287 - self._run_workers("init_model", cupy_port=get_open_port())
287 + # don't use cupy for eager mode
288 + self._run_workers("init_model",
289 + cupy_port=get_open_port()
290 + if not model_config.enforce_eager else None)
291 self._run_workers(
292 "load_model",
293 max_concurrent_workers=self.parallel_config.