@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 464dd985e3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix num_gpus when TP > 1 (#1852)

Woosuk Kwon committed Dec 3, 2023 at 12:24 UTC 464dd985e31a62cb7f87ab5b710a50db1912d177
2 files changed +15 -2
vllm/engine/async_llm_engine.py
+10 -1
@@ -301,7 +301,16 @@ class AsyncLLMEngine:
301 elif self.worker_use_ray:
302 engine_class = ray.remote(num_cpus=0)(self._engine_class).remote
303 else:
304 - engine_class = ray.remote(num_gpus=1)(self._engine_class).remote
304 + # FIXME(woosuk): This is a bit hacky. Be careful when changing the
305 + # order of the arguments.
306 + cache_config = args[1]
307 + parallel_config = args[2]
308 + if parallel_config.tensor_parallel_size == 1:
309 + num_gpus = cache_config.gpu_memory_utilization
310 + else:
311 + num_gpus = 1
312 + engine_class = ray.remote(num_gpus=num_gpus)(
313 + self._engine_class).remote
314 return engine_class(*args, **kwargs)
315
316 async def engine_step(self) -> bool:
vllm/engine/llm_engine.py
+5 -1
@@ -159,9 +159,13 @@ class LLMEngine:
159 for bundle in placement_group.bundle_specs:
160 if not bundle.get("GPU", 0):
161 continue
162 + if self.parallel_config.tensor_parallel_size == 1:
163 + num_gpus = self.cache_config.gpu_memory_utilization
164 + else:
165 + num_gpus = 1
166 worker = ray.remote(
167 num_cpus=0,
164 - num_gpus=self.cache_config.gpu_memory_utilization,
168 + num_gpus=num_gpus,
169 scheduling_strategy=PlacementGroupSchedulingStrategy(
170 placement_group=placement_group,
171 placement_group_capture_child_tasks=True),