A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix num_gpus when TP > 1 (#1852)
Woosuk Kwon committed
Dec 3, 2023 at 12:24 UTC
464dd985e31a62cb7f87ab5b710a50db1912d177
2 files changed
+15
-2
vllm/engine/async_llm_engine.py
+10
-1
@@ -301,7 +301,16 @@ class AsyncLLMEngine:
301
elif self.worker_use_ray:
302
engine_class = ray.remote(num_cpus=0)(self._engine_class).remote
303
else:
304
- engine_class = ray.remote(num_gpus=1)(self._engine_class).remote
304
+ # FIXME(woosuk): This is a bit hacky. Be careful when changing the
305
+ # order of the arguments.
306
+ cache_config = args[1]
307
+ parallel_config = args[2]
308
+ if parallel_config.tensor_parallel_size == 1:
309
+ num_gpus = cache_config.gpu_memory_utilization
310
+ else:
311
+ num_gpus = 1
312
+ engine_class = ray.remote(num_gpus=num_gpus)(
313
+ self._engine_class).remote
314
return engine_class(*args, **kwargs)
315
316
async def engine_step(self) -> bool:
vllm/engine/llm_engine.py
+5
-1
@@ -159,9 +159,13 @@ class LLMEngine:
159
for bundle in placement_group.bundle_specs:
160
if not bundle.get("GPU", 0):
161
continue
162
+ if self.parallel_config.tensor_parallel_size == 1:
163
+ num_gpus = self.cache_config.gpu_memory_utilization
164
+ else:
165
+ num_gpus = 1
166
worker = ray.remote(
167
num_cpus=0,
164
- num_gpus=self.cache_config.gpu_memory_utilization,
168
+ num_gpus=num_gpus,
169
scheduling_strategy=PlacementGroupSchedulingStrategy(
170
placement_group=placement_group,
171
placement_group_capture_child_tasks=True),