A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Don't use cupy when `enforce_eager=True` (#3037)
Roy committed
Feb 27, 2024 at 09:33 UTC
c1c0d00b88320f97e00a3175fac235a232893da5
1 file changed
+4
-1
vllm/engine/llm_engine.py
+4
-1
@@ -284,7 +284,10 @@ class LLMEngine:
284
is_driver_worker=True,
285
)
286
287
- self._run_workers("init_model", cupy_port=get_open_port())
287
+ # don't use cupy for eager mode
288
+ self._run_workers("init_model",
289
+ cupy_port=get_open_port()
290
+ if not model_config.enforce_eager else None)
291
self._run_workers(
292
"load_model",
293
max_concurrent_workers=self.parallel_config.